Background
I have a NodeJS app that is meant to be used as a RESTful API. It is connected with a MongoDB database in the backend using Mongoose. The app is built upon the idea of nested documents. It stores wikis, sections and notes with the following schema:
const noteSchema = new mongoose.Schema({ title: String, content: String });
const sectionSchema = new mongoose.Schema({ title: String, notes: [noteSchema] });
const wikiSchema = new mongoose.Schema({ title: String, sections: [sectionSchema] });
All of which are accessed via a single model of the wiki:
const wikiModel = mongoose.model("Wiki", wikiSchema);
A user can do GET, POST, PUT, DELETE requests on each of the endpoints to manipulate the data inside. If someone wants to ping the Notes endpoint (the furthest down in the hierarchy), it must first check the wiki and then the section endpoint, to ensure that each of them exists.
Here's an example:
app.get('/:wikiTitle/:sectionTitle/:noteTitle', function(req, res) {
wikiModel.findOne({ title: req.params.wikiTitle }, function(err, wiki) {
if (err) {
res.send('\nAn unkown error has occured');
console.error(err);
} else if (wiki) {
const sectionTitle = req.params.sectionTitle;
wikiModel.findOne({ 'sections.title': sectionTitle }, function(err, section) {
if (err) {
res.send('\nAn unkown error has occured');
console.error(err);
} else if (section) {
const noteTitle = req.params.noteTitle;
wikiModel.findOne({ 'sections.notes.title': noteTitle }, function(err, n) {
if (err) {
res.send('\nAn unkown error has occured');
console.error(err);
} else if (n) {
const section = n.sections.find((s) => { return s.title === sectionTitle; });
const note = section.notes.find((n) => { return n.title === noteTitle; });
if (note.content) {
res.send('\n' + note.title + '\n\n' + note.content);
} else {
res.send('\n' + note.title + '\n\n[ No content to show ]');
}
} else {
res.send('\nNo such note exists');
}
});
} else {
res.send('\nNo such section exists');
}
});
} else {
res.send('\nNo such wiki exists');
}
});
});
This is a very lengthy method and the first two queries are actually frequently throughout the app. I also understand a MongoDB query is an asynchronous operation and thus, why I put each consequent MongoDB query within it's parent (the one I wish to finish before that one begins).
Question
Is there a way to split each MongoDB query into its own method or introduce promises in a way that would shorten the code? I would rather prefer advice that ultimately causes the splitting of my code into individual methods as what you see above is one of many endpoints which all use the same queries.
So in the end result I would like to have something close to the likes of:
app.get('/:wikiTitle/:sectionTitle/:noteTitle', function(req, res) {
if (getWiki(req.params.wikiTitle)) {
// Continue with second query
if (getSection(req.params.sectionTitle)) {
// Continue with third query...
}
}
});
function getWiki(wikiTitle) {
wikiModel.findOne({ title: wikiTitle }, function(err, wiki) {
if (err) {
console.error(err);
res.send('An unknown error occured.');
} else if (wiki) {
// Send OK result to continue to next query
return wiki
} else {
res.send('No wiki found');
return null;
}
});
}
function getSection(sectionTitle) {
wikiModel.findOne({ 'sections.title': sectionTitle }, function(err, section) {
if (err) {
console.error(err);
res.send('An unknown error occured.');
} else if (section) {
// Send OK result to continue to next query
return section
} else {
res.send('No section found');
return null;
}
});
}
I am hoping this will significantly cut the length of code and also utilise re-usability of code. Any advice on how I could come close to achieving something like this is welcome.
You can definitely use callbacks in the same way as the ones call your model. For example:
app.get('/:wikiTitle/:sectionTitle/:noteTitle', function(req, res) {
getWiki(req.params.wikiTitle, function (err, title) {
if (err) {
return res.send(err);
}
getSection(req.params.sectionTitle, function (err, section) {
if (err) {
return res.send(err);
}
// Todo: use title and section, etc...
});
});
});
function getWiki(wikiTitle, cb) {
wikiModel.findOne({ title: wikiTitle }, function(err, wiki) {
if (err) {
console.error(err);
return cb('An unknown error occured.');
} else if (wiki) {
// Send OK result to continue to next query
return cb(null, wiki);
} else {
return cb('No wiki found');
}
});
}
function getSection(sectionTitle, cb) {
wikiModel.findOne({ 'sections.title': sectionTitle }, function(err, section) {
if (err) {
console.error(err);
return cb('An unknown error occured.');
} else if (section) {
// Send OK result to continue to next query
return cb(null, section);
} else {
return cb('No section found');
}
});
}
This is a standard way of using async functions in node. By convention, the first parameter is always an error parameter.
If you want your code to be cleaner, you can try to use guard clauses / early outs to exit error cases early. This will cut down on your need for if / else conditional statements.
You can also look into libraries like async for cleaner chaining of asynchronous calls.
When you are comfortable, you can also look into using promises and the 'async' javascript keyword (different from the async library above, confusing, I know) which will also allow you to cut down on the lines of code you have to write to get nice async code.
You should use async functions (Promises) like
app.get('somePath', async (req, res, next) => {
try {
const doc = await model.find({ someField: 'some value' }).exec(); // exec returns promise
res.send({ document: doc });
} catch (error) {
// here you can handle all errors or/and call next for the error middleware
next(error);
}
});
Related
This is my first time asking a question on stackoverflow. Sorry if I made posting mistakes.
I am trying to exit a function after sending a response to prevent continuing through the function.
node -v = v12.6.0
express = ^4.17.1
mongoose = ^5.6.6
// handle adding a new book request submission
addNewBook: function (req, res) {
var response = null;
var name = req.body.name.toLowerCase();
var url = req.body.url.toLowerCase();
var category = req.body.category.toLowerCase();
var tags = req.body.tags.toLowerCase();
// checking if category already exist. if not, make a new category
Category.find({label: category}).exec(function(err, data) {
if(err) {
response = res.status(400).send({message:'Error finding category.'});
} else if(data.length === 0) {
var newCategory = new Category({label: category, description: '', keywords: ''});
newCategory.save(function(err, data){
if(err) {
response = res.status(400).send({message:'Error saving new category.'});
}
})
}
});
// checking if book name already exist
Book.find({name: name}).exec(function(err, data){
if(err) {
response = res.status(400).send({message:'Error validating Book existence'});
} else if(data.length > 0) {
response = res.status(200).send({message:'book name already exist'});
} else {
req.body.name = name;
req.body.url = url;
req.body.category = category;
req.body.tags = tags;
// make a new book document
var newBook = new Book(req.body);
newBook.save(function (err, data) {
if (err) {
response = res.status(400).send({message: 'Error saving new Book.'});
} else {
response = res.json(data);
}
})
}
});
return response;
},
Function continues to executes other part of the function code after a return.
I am also getting "Cannot set headers after they are sent to the client" error on node. Im guessing, preventing the function to continue after sending a response will fix this as well?
Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client
There are two problems with the flow of your logic. First is that return only returns a function. It does not return the function that calls a function or the function that defines a function.
Basically, your code is:
Category.find({label: category}).exec(function(err, data) {
if(err) {
// ...
return;
} else if(/* ... */) {
// ...
newCategory.save(function(err, data){
if(err) {
// ...
return;
}
})
}
});
moreStuffDownHere();
// ...
Let's rewrite that to not use anonymous functions to make it clear what's really happening
function findCallback (err, data) {
if(err) {
// ...
return; // it's obvious that this returns form findCallback()
// and NOT yourFunction()
} else if(/* ... */) {
// ...
newCategory.save(saveCallback);
}
}
function saveCallback (err, data) {
if(err) {
// ...
return;
}
}
function yourFunction () {
Category.find({label: category}).exec(findCallback);
moreStuffDownHere();
}
So you can now see that you are not calling return anywhere in yourFunction().
The second problem is that Category.find().exec() is asynchronous. This means it returns immediately and let any code below it run before calling findCallback(). To solve the async issue just move moreStuffDownHere() inside findCallback().
Therefore, the simplest change to get your program flow working is to move moreStuffDownHere:
Category.find({label: category}).exec(function(err, data) {
if(err) {
res.status(400).send({message: 'Error finding category.'});
return;
} else if(data.length === 0) {
var newCategory = new Category({label: category, description: '', keywords: ''});
newCategory.save(function(err, data){
if(err) {
res.status(400).send({message: 'Error saving new category.'});
return;
}
// More stuff down here, that now will only execute if there are no errors
})
}
});
Improve program flow
One issue I have with the solution above is that now moreStuffDownHere is hardcoded inside the save callback. One way around it is to refactor the entire operation and make it your own internal API:
function addNewCategory (category, callback) {
// callback will be passed status depending on success or failure
Category.find({label: category}).exec(function(err, data) {
if(err) {
// ...
callback('FIND_FAILURE');
return;
} else if(/* ... */) {
// ...
newCategory.save(function(err, data){
if(err) {
// ...
callback('SAVE_FAILURE');
return;
}
callback('OK');
})
}
});
}
Now inside yourFunction() you can check the result of the entire operation and decide to return or continue:
function yourFunction() {
// ...
addNewCategory(category, function (status) {
switch (status) {
case 'FIND_FAILURE':
res.status(400).send({message: 'Error finding category.'});
return;
case 'SAVE_FAILURE':
res.status(400).send({message: 'Error saving new category.'});
return;
}
// More stuff down here ...
});
}
Improvement 2 - Promises
It's possible to make the program flow much easier to read by using Promises along with async/await. For that you need to wrap the operation in a promise. We can use the addNewCategory function we wrote above as an example:
function addNewCategory (category) {
// returns a Promise of true/false
return new Promise(function (resolve, reject) {
Category.find({label: category}).exec(function(err, data) {
if(err) {
// ...
resolve('FIND_FAILURE'); // you can also use reject if you want
// to use a try/catch flow
return;
} else if(/* ... */) {
// ...
newCategory.save(function(err, data){
if(err) {
// ...
resolve('SAVE_FAILURE');
return;
}
resolve('OK');
})
}
});
});
}
Now the code is slightly easier to follow because it allows you to keep moreStuffDownHere where you originally have it without moving it inside another function:
async function yourFunction() {
// ...
var status = await addNewCategory(category);
switch (status) {
case 'FIND_FAILURE':
res.status(400).send({message: 'Error finding category.'});
return;
case 'SAVE_FAILURE':
res.status(400).send({message: 'Error saving new category.'});
return;
}
// More stuff down here ...
}
Note: Express accepts functions marked as async as routes/middlewares. You just need to call res.send() or next() as usual
The error is as a result of your condition. Hence, both code blocks are run resulting in the response being sent twice. To fix this change your code to this below.
Category.find({label: category}).exec(function(err, data) {
if(err) {
res.status(400).send({message: 'Error finding category.'});
} else if(data.length>0) {
//there is no data with that label - Hence, create one
var newCategory = new Category({label: category, description: '', keywords: ''});
newCategory.save(function(err, data){
if(err) {
//if error
res.status(400).send({message: 'Error saving new category.'});
}else{
//if item saves
res.status(200).send({message: 'Item saved'});
}
})
}else{
//there is a data with that label availble - do something else
res.status(200).send(data)
}
});
The error you report happens when there are code paths that can send a response more than once. You get one and only one response per request. So, calling res.send() more than once is one way that you get that error.
Preventing this when you have a number of asynchronous operations requires a bit more work as you have to make sure that all your code is properly sequenced and all error paths are properly terminated (so further processing doesn't happen). In general, this code is a lot easier to write using promise-based interfaces for your asynchronous operations, but since you aren't using the promise interface on your database, I'll show how you can do it with your existing callback interface. In generally, it involves a lot of nesting inside of asynchronous callbacks and very careful if/else and return around conditionals and errors.
Your code is subject to this error because you are running Category.find() and Book.find() in parallel. You don't wait for the Category.find() code to finish before doing the book operations. If the category code causes an error, you will send that error response, but still continue with the book code which will then send its response. Instead, you need to make sure that if there's an error with the category stuff that you don't run the book code at all. For the plain callback interface on your database, that means nesting the book code inside a callback from the category code. To make this simpler to write, I put the category code into it's own function that has one completion callback that we can use to know when its all done.
Here's one way to do it:
// utility function to create category if needed, requires callback
// to communicate results
function createCategoryIfNeeded(category, fn) {
// checking if category already exist. if not, make a new category
Category.find({label: category}).exec(function(err, data) {
if(err) {
fn({message:'Error finding category.'});
} else if(data.length === 0) {
let newCategory = new Category({label: category, description: '', keywords: ''});
newCategory.save(function(err, data){
if (err) {
fn({message:'Error saving new category.'});
} else {
// category created
fn(null, true);
}
})
} else {
// category already exists
fn(null, false);
}
});
}
// handle adding a new book request submission
addNewBook: function (req, res) {
var name = req.body.name.toLowerCase();
var url = req.body.url.toLowerCase();
var category = req.body.category.toLowerCase();
var tags = req.body.tags.toLowerCase();
createCategoryIfNeeded(category, function(err, created) {
if (err) {
res.status(400).send(err);
} else {
// checking if book name already exist
Book.find({name: name}).exec(function(err, data){
if(err) {
res.status(400).send({message:'Error validating Book existence'});
} else if(data.length > 0) {
res.status(200).send({message:'book name already exist'});
} else {
req.body.name = name;
req.body.url = url;
req.body.category = category;
req.body.tags = tags;
// make a new book document
var newBook = new Book(req.body);
newBook.save(function (err, data) {
if (err) {
res.status(400).send({message: 'Error saving new Book.'});
} else {
res.json(data);
}
});
}
});
}
});
},
The error meassage says that, res can be send once it has been send. So returning it along with the response.
Category.find({label: category}).exec(function(err, data) {
if(err) {
return res.status(400).send({message: 'Error finding category.'});
} else if(!data) {
var newCategory = new Category({label: category, description: '', keywords: ''});
newCategory.save(function(err, data){
if(err) {
return res.status(400).send({message: 'Error saving new category.'});
}
})
}
});
I don't have much experience in async code and I'm stuck with a problem, here is my code :
if (!req.params.user_name) {
req.params.user = req.me;
} else {
User.findOne({username: req.params.user_name}, '_id', (error, user) => {
if (error) {
return next(error);
}
if (!user) {
let error = new Error('User not found');
return next(error);
}
req.params.user = user;
});
}
Account.findOne({name: req.params.account_name, created_by: req.params.user})
.populate(['currency', 'created_by'])
.exec((err, account) => {
if (err) {
return next(err);
}
return res.send(account);
});
As you can see the problem is that in one case I just have a simple procedural action to do, in the other I have to query the database which is async, then I have to execute the code below. I can't just simply put the Account query in the callback of the User query because I don't need to execute User query all the time.
I've tried to find an answer here but the only results I've found are about executing one async task or another (ex: Working with promises inside an if/else).
Following the recommandations on this post I've thought about wrapping the code inside the if block in an anonymous function and do something like:
let get_user;
if (!req.params.user_name) {
let get_user = () => {req.params.user = req.me};
} else {
let get_user = User.findOne({username: req.params.user_name}, '_id', (error, user) => {
if (error) {
return next(error);
}
if (!user) {
let error = new Error('User not found');
return next(error);
}
req.params.user = user;
});
}
get_user().then(() => {
Account.findOne({name: req.params.account_name, created_by: req.params.user})
.populate(['currency', 'created_by'])
.exec((err, account) => {
if (err) {
return next(err);
}
return res.send(account);
});
});
But I find it weird and I guess I would need to return a Promise from the anonymous function in the if block.
So what would be an elegant way of solving this ? I'm learning node.js on my free time and any general suggestions would be greatly appreciated.
Thanks for your time.
Callbacks shouldn't be used with Mongoose; it has been supporting promises for a long time.
This
let get_user = () => {req.params.user = req.me};
won't work because get_user is expected to return a promise.
It's usually done like:
let userPromise;
if (!req.params.user_name) {
userPromise = Promise.resolve(req.me);
} else {
userPromise = User.findOne(...);
}
userPromise().then((user => {
req.params.user = user;
return Account.findOne(...);
});
Notice that req.params.user is common code for both conditions. This is conveniently done with async..await, which is syntactic sugar for promises:
try {
let user;
if (!req.params.user_name) {
user = req.me;
} else {
user = await User.findOne({username: req.params.user_name}, '_id');
}
req.params.user = user;
const account = await Account.findOne({name: req.params.account_name, created_by: req.params.user})
.populate(['currency', 'created_by'])
.exec();
res.send(account);
} catch (err) {
next(err);
}
This is supposed to be a body of async middleware function, which wasn't shown in original code.
As explained in this answer, Express doesn't support promises itself, all async middlewares and route handlers should be wrapped with try..catch for error handling.
Apologies if I'm just being thick. I've tried the search function, but being relatively new to all of this, I'm struggling to work out the solution. I think I'm probably not searching for the right keyword.
I have a route in my Node.js application that has two forEach loops in it. I want forEach loop 1 to finish, then start forEach loop 2. When that finishes, I then want to call my res.redirect. Currently the route is going straight to res.redirect, and doesn't appear to be completing the forEach loops.
Code:
// Auto-populate entries
router.post("/populate", middlewareObj.isLoggedIn, function(req, res) {
var baseData = []
//lookup Plan using ID
Plan.findById(req.params.id, function(err, foundPlan) {
if (err) {
console.log(err);
res.redirect("/plans");
} else {
BaseData.find({
"contributingRegion": foundPlan.contributingRegion
}, function(err, foundRecords) {
foundRecords.forEach(function(record) {
baseData.push(record)
baseData.save
});
//Create entries & push into plan
baseData.forEach(function(data) {
if (includes(req.body.orgs, data.org)) {
Entry.create(data, function(err, entry) {
if (err) {
console.log(err);
} else {
entry.author.id = req.user._id;
entry.author.username = req.user.username;
entry.save();
foundPlan.planEntries.push(entry);
foundPlan.save();
}
})
}
})
res.redirect('/plans/' + foundPlan._id);
});
}
});
});
There are many ways to achieve this, for example you could use promises or async module, you could also use recurrent functions, I will provide a solution with the async module because it let you understand how asynchronous functions work and how to control them:
async.each( baseData, function (data, next) {
if (includes(req.body.orgs, data.org)) {
Entry.create(data, function(err, entry) {
if (err) {
// stop iterating and pass error to the last callback
next(err);
} else {
entry.author.id = req.user._id;
entry.author.username = req.user.username;
entry.save();
foundPlan.planEntries.push(entry);
foundPlan.save();
// next iteration
next();
}
});
} else {
// next iteration
next();
}
}, function (err) {
// This function runs when all iterations are done
if (err) throw err;
res.redirect('/plans/' + foundPlan._id);
} );
I have a written an api in nodejs to find list of leads or customers. Now i have to use this api from another controller. How to pass query paramters from another controller and get list of leads from it. Hence will be able to reuse code.
exports.listofLeads = function (req, res) {
var param = req.query.from; var s = "initialSource"; var queryLeads = Customers.find({"attributes": { $size: 0 }} ,{"email":1}); if(! param) {
queryLeads.exec(function (err, articles) {
if (err) {
return res.status(422).send({
message: errorHandler.getErrorMessage(err)
});
} else {
res.json(articles);
}
}); } else {
queryLeads.and([ { [s]: param } ]).exec(function (err, articles) {
if (err) {
return res.status(422).send({
message: errorHandler.getErrorMessage(err)
});
} else {
res.json(articles);
}
});
}
};
Using require() you can access file in which you have written this function. So in other api just call this function and pass parameters as you did in first api
I need to query 2 different collections and send it in the express response. I have a very vague idea of what is needed to do so. I tried to contact the query documents to an empty array and send that new array as the response. But I receive an empty array as a response.
This is my route.
site.route('/campus/development')
.get(function(req, res) {
var devPosts = [];
development.find().exec(function(err, docs) {
if (err) {
console.log('Error : ' + err);
} else {
if (docs != null) {
devPosts = devPosts.concat(docs);
console.log(docs);
} else {
console.log('No posts found');
}
}
});
jobs.find().exec(function(err, jobs) {
if (err) {
console.log('Error : ' + err);
} else {
if (jobs != null) {
devPosts = devPosts.concat(jobs);
console.log(jobs);
} else {
console.log('No jobs');
}
}
});
res.send(devPosts);
});
This is due to the async operation of the requests to the database. There are a variety of solutions to this but basically distill down to two types: callbacks or promises.
A callback solution might look like:
site.route('/campus/development')
.get(function(req, res) {
development.find().exec(function(err, devDocs) {
if (err) {
console.log('Error : ' + err);
} else {
if (devDocs != null) {
console.log(devDocs);
jobs.find().exec(function(err, jobs) {
if (err) {
console.log('Error : ' + err);
} else {
if (jobs != null) {
console.log(jobs);
res.send([devDocs, jobs]);
} else {
console.log('No jobs');
}
}
});
} else {
console.log('No posts found');
}
}
});
});
But this introduces a couple of interesting issues: one is the phenomenon known as callback hell and the other is that you should be responding with the errors which means you would need to have a response call for each error (albeit this is a very simplistic approach to it).
As mentioned earlier there is another type of solution which involves using promises. There are a bunch of libraries that you can use and actually Mongoose returns a promise from the exec method. However if you are on Node 0.12.x you can also use the native Promise (it was introduced in 0.11 but you should be using 0.12.x over 0.11.x). A benefit to using the native promise over the one returned from Mongoose is that you can execute these requests in parallel since they don't depend on each other.
site.route('/campus/development')
.get(function(req, res) {
Promise.all([
development.find().exec(), // returns a promise
jobs.find().exec() // returns a promise
]).then(function(results) {
// results is [devDocs, jobs]
console.log(results);
res.send(results);
}).catch(function(err) {
res.send(err);
});
});