Mongoose Async Call Issue with Find Query - node.js

I have var movieRecommendation which is being populated from data coming from Mongo DB. Issue is Mongoose Movie.findOne() call is asycn call which is not allowing me to get my final populated movieRecommendation which I need to send back as response.
exports.getRecommendation=function(req,res){
var movieRecommendation = [];
var id=req.params.id;
console.log('----- Get User Recommendation - ' + id);
var url = 'http://52.8.48.113:8080/recommender-server/recommender/v1/recommendations/'+id+'.do';
//make http get request
request({
url: url,
json: true
}, function (error, response, recommendations) {
// res.json(recommendations);
if (!error && response.statusCode === 200) {
recommendations.forEach(function(entry) {
**Movie.findOne({'id':parseInt(entry.itemId)},function(err, movieData){**
entry.movie = movieData;
movieRecommendation.push(entry);
//console.log('rec', movieRecommendation);
console.log(movieRecommendation.length);
});
});
}
console.log("====Final========"+movieRecommendation.length);
//Output = 0
});
res.json(movieRecommendation); // Here movieRecommendation is coming as black Array
};
Please let me know how I can get finally populated movieRecommendation var at end to make it available for response.

For this type of issues we can use Async library. To populate the data finally once all the operations done, we can use async.each collection from Async Library.
For example:
NOTE:Install Async by this command
npm install async to use async library
var async = require("async");
var recomenmendations = [{"data2" : "value2"} , {"data1" : "value2"}, {"data3" : "value3"}, {"data4" : "value4"} ]
var movieRecommendation = [];
async.each(recomenmendations,
function(recomenmendationItem, callback){
console.log("Here you can query the required data using current recomenmendations ITEM");
console.log(recomenmendationItem);
callback();
// Movie.find({'id':parseInt(recomenmendationItem.itemId)},function(err, movieData){
// recomenmendationItem.movie = movieData;
// movieRecommendation.push(entry);
// callback();
// });
},
function(err){
console.log("here you can send your resopnse");
console.log("This section will be executed once all the recomenmendations are processed");
//res.json(movieRecommendation)
}
);
You can query the mongoDB as shown with comment section. You should use callback() once all the operations performed for an iteration.

As I mentioned in one of my comments, use the callback passed to the iterator function and call it inside the Movie.findOne() callback. That way, async.each will know when each step has finished:
async.each(recomendations, function (recomendationItem, callback) {
Movie.findOne({'id':parseInt(entry.itemId)},function(err, movieData){
if (err) return callback(err); // if you have an error on you search, just pass it to the iterator callback
recommendationItem.movie = movieData;
movieRecommendation.push(recommendationItem);
callback();
});
}, function (error) {
if (error) return res.json ({ error: error }); // you should also check if an error ocurred
res.json(movieRecomendation);
});
Just to point out: you can also use async.eachSeries, that will just call the next step of your iteration when the previous one has returned (if that matters to you, but I think it's not your case though) and it has the same signature.

#Vivek Panday replace the following code inside your exports.getRecommendation function to get your expected output. We don't need to use the count variable if we use the callback function. And an important thing is we have to use callback(); once all the process done. I think you have not used callback function properly in The example you have worked out. Use the following code If there is any issue please let me know.
var async = require('async');
var request = require("request");
var movieRecommendation = [];
var id=req.params.id;
console.log('----- Get User Recommendation - ' + id);
var url = 'http://52.8.48.113:8080/recommender-server/recommender/v1/recommendations/'+id+'.do';
//make http get request
request({
url: url,
json: true
}, function (error, response, recommendations) {
if (!error && response.statusCode === 200) {
console.log('recommendation lenght '+ recommendations.length);
async.each(recommendations,
function(recommendationItem, callback){
Movie.findOne({'id':parseInt(recommendationItem.itemId)},function(err, movieData){
recommendationItem.movie = movieData;
movieRecommendation.push(recommendationItem);
//you have to use callback(); once all your process is done
callback();
});
},
function(err){
//you should use this function, this will be execute once all the process done
console.log(movieRecommendation);
console.log("finally callback");
res.json(movieRecommendation);
}
);
}
});

I have tried as per given suggestion above ..
var async = require("async");
var recomenmendations = [{"data2" : "value2"} , {"data1" : "value2"}, {"data3" : "value3"}, {"data4" : "value4"} ]
var movieRecommendation = [];
async.each(recomenmendations,
function(recomenmendationItem, callback){
console.log("Here you can query the required data using current recomenmendations ITEM");
console.log(recomenmendationItem);
// Movie.find({'id':parseInt(recomenmendationItem.itemId)},function(err, movieData){
recomenmendationItem.movie = movieData;
movieRecommendation.push(entry);
console.log("any data"); // line y
});
callback();
},
function(err){
console.log("here you can send your resopnse"); // line x
console.log("This section will be executed once all the
recomenmendations are processed");
//res.json(movieRecommendation)
}
);
But still face same issue line x is printing before line y ,which is making again same issue.
However I have tried something given below and achieved expected result .
exports.getRecommendation=function(req,res){
var movieRecommendation = [];
var id=req.params.id;
console.log('----- Get User Recommendation - ' + id);
var url = 'http://52.8.48.113:8080/recommender-server/recommender/v1/recommendations/'+id+'.do';
//make http get request
request({
url: url,
json: true
}, function (error, response, recommendations) {
// res.json(recommendations);
if (!error && response.statusCode === 200) {
console.log('recommendation lenght '+ recommendations.length);
// recommendations.forEach(function(entry) {
var count=0;
async.each(recommendations,function(recommendationItem){
// console.log(recommendationItem);
Movie.findOne({'id':parseInt(recommendationItem.itemId)},function(err, movieData){
recommendationItem.movie = movieData;
movieRecommendation.push(recommendationItem);
count ++;
console.log('final res length : ' + movieRecommendation.length);
console.log('final res length count : ' + count +' and item recomm lenght ' + recommendations.length );
if(count === recommendations.length){
console.log(' =====Final=====> here you can send your response =========' + movieRecommendation.length);
res.json(movieRecommendation);
}
});
// callback();
});
}
});
};
Still I am open for any feedback and suggestions.

Related

NodeJs : Using async for request url - Post method sending response before execution of all urls

I have a post method, whose request input is a list of newLink objects (newLink with attributes linkUrl and status)
I am using async.map for iterating over my URLs to check if the links are active or not.
newLinks contains links like {www.google.com,www.nourl.com,www.xyz.com} I am expecting like after all the request are processed and setting corresponding status as true or false, I want this to send using res.send(newLinks)
But the console is giving the below results: "www.google.com is up", then calling res.send(), then executing "www.nourl.com is up" and "www.xyz.com is up"
So basically here, after the first url request , my code below is executing the function outside the async loop. I thought async will only allow the next piece of code to execute only after all the urls are validated.
app.post('/myposturl', function(req , res){
var request = require('request');
let linkDetails= req.body.linkDetails;
var i = 0;
async.map(linkDetails, function(newLink, callback) {
var Url = "url";
var url = newLink.linkUrl;
var proxiedRequest = request.defaults({'proxy': Url});
proxiedRequest(url , function (error, response, body) {
if(error){
console.log('Err: '+ error);
}
if (!error) {
if(response.statusCode == 200 || response.statusCode == 201 ||
response.statusCode == 202){
console.log(url + ' is up!!');
newLink.isActive = true;
}
if(response.statusCode == 301 || response.statusCode == 302){
console.log(url + ' is redirecting us!!');
return false;
}
}
});
callback();
} , function(err, linkDetails) {
res.send(linkDetails);
});
//tried res.send here as well.
});
}
The callback of async.map should invoke inside proxiedRequest. What your code is doing now: invoke callback immediately before the proxiedRequest finished. Also return false; does not work in asynchronous function. You should return the new status like this callback(null, newLink). After all the request are processed, the newLinkDetails will be the array of all newLink.
Note, that since this function applies the iteratee to each item in parallel, there is no guarantee that the iteratee functions will complete in order.
If you need to keep the order, user mapSeries insted.
Please read the doc of async.map for more. Hope it helps.
app.post('/myposturl', function(req , res){
//other codes
async.map(linkDetails, function(newLink, callback) {
//other codes
proxiedRequest(url , function (error, response, body) {
if(error){
console.log('Err: '+ error);
callback(error);
//^ ^ ^ ^ ^
// Validation failed, return from here
}
else {
//some validation & set newLink.isActive
callback(null, newLink);
// ^ ^ ^ ^ ^
//return newLink status by invoking the callback
}
});
}, function(err, newLinkDetails) {
//err = if any Validation failed
// now all the request are processed,newLinkDetails is array of all newLink's
res.send(newLinkDetails);
});
});
Usually when using async.js, I follow these two principles:
Always call the callback at least AND at most once during your async function.
Call the callback only when the async function is complete OR if an error has occurred. If the latter occurs, call the callback passing the error AND stop further execution of the async function e.g. return callback(error)
I would revise your code as below:
var request = require('request');
app.post('/myposturl', function (req , res) {
async.mapSeries(req.body.linkDetails || [], function(newLink, callback) {
var Url = "url";
var proxiedRequest = request.defaults({ 'proxy': Url });
proxiedRequest(newLink.linkUrl, function (err, response, body) {
if (err)
return callback(err);
// I'm assuming you don't want to stop checking the links for bad status codes
if ([301, 302].indexOf(response.statusCode) > -1){
return callback(null, url + ' is redirecting us!!');
if ([200, 201, 202].indexOf(response.statusCode) == -1) {
return callback(null, url + ' came back with ' + response.statusCode);
console.log(url + ' is up!!');
newLink.isActive = true;
callback(null, newLink);
});
}, function (err, linkDetails) {
// when all links get checked, it will come down here
// or if an error occurs during the iteration, it will come down here
console.log(err, linkdetails);
res.send(linkDetails);
});
});
If you only want to get back active links, you may also want to check out async.filterSeries(). Here the callback would need to be passed a boolean in its second argument.

Calling a module's local function as callback in "request"

In my main code, I do the following:
var module = require('./module')
module.FooA(module.FooB);
module.js contains the next code:
var request = require('request'); //using of npm "request"
exports.FooB = function(data){ /*operations on data here*/ };
exports.FooA = function(callback){
var url = some_link;
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
callback(body);
};
});
};
The issue is that apparently, callback(body) doesn't run even if the conditions meet. var result = request(url) followed by exports.FooB(result) does the job, but as far as I can see, obviously does not act like a callback, and would produce troubles.
What is the proper way of defining a callback function in such a case? Do I need at all, or it is actually synchronous and I missed to notice it?
Use first function callback params with error, this is an default in node.js core and is google for your project functions.
And like #ShanSan commend, use console.log, console.error or console.trace for debug.
Example:
var request = require('request'); //using of npm "request"
exports.FooB = function(error, data){ /*operations on data here*/ };
exports.FooA = function(callback){
var url = some_link;
request(url, function (error, response, body) {
if (error || response.statusCode != 200) {
// pass error to callback and if use return you dont need other code block bellow
console.error('Error in request', error);
return callback(error, null);
}
// here run if dont have errors
// if need more info use the console.log(request); or console.log(body);
// use error in first param in callback functions
callback(null, body);
});
};

How to handle node http requests

I'm trying to understand how to wait for http requests to finish in node. I want to make two http requests and use the results in a function that gets called after the http requests are finished.
I'm using async and request and have been using async.series as following:
var request = require("request");
var express = require('express');
var async = require("async");
app.get('/rx', function(req, res) {
var drug1 = req.query.drug1;
var drug2 = req.query.drug2;
console.log("in rx")
console.log(drug1);
console.log(drug2);
var id1 = '';
var id2 = '';
/*part of code I'm concerned with*/
async.series([
function(callback) {
id1 = getID(drug1);
console.log("function one");
console.log(id1);
callback();
},
function(callback) {
id2 = getID(drug2);
console.log("function two");
console.log(id2);
callback();
}
],
function(err, results) {
console.log(id1);
console.log(id2);
request("http://rxnav.nlm.nih.gov/REST/interaction/interaction.json?list?rxcuis=" + id1 + "&sources=" + id2, function(error, response, body) {
console.log("finished!");
res.json(body);
});
});
});
//returns an int ID
function getID(drugName) {
request("http://rxnav.nlm.nih.gov/REST/Prescribe/rxcui.json?name=" + drugName, function(error, response, body) {
var content = JSON.parse(body);
var id = parseInt(content.idGroup.rxnormId);
console.log("in getID function");
console.log(id);
return id;
});
}
The console output shows:
in rx
advil
ibuprofen
seriesone
undefined
two
undefined
undefined
undefined
finished!
GET /rx?drug1=advil&drug2=ibuprofen 304 345ms
in getID function
153010
in getID function
5640
I want to wait until each http request function is completed, and then proceed to the next portion of code. How do I achieve this?
This question (or variants thereof) has been asked more than 1000 times here on StackOverflow. Therefore I'm not going to explain it but you can search "return async" on this site (the search input at the top right corner) if you want to know more.
The basic problem is that it's impossible to return values from an async function (ever wonder why they accept callbacks?).
For your specific case, you need to change getId() to:
//returns an int ID
function getID(drugName, callback) {
request("http://rxnav.nlm.nih.gov/REST/Prescribe/rxcui.json?name=" + drugName, function(error, response, body) {
var content = JSON.parse(body);
var id = parseInt(content.idGroup.rxnormId);
console.log("in getID function");
console.log(id);
callback(null,id); // <--- this is how you return async values
});
}
Note: the null is because functions in the async.js family expects the first argument to the callback to be an error. So if there are no errors pass null. This by the way is a standard node.js practice.
Then inside async.series() you do:
async.series([
function(callback) {
getID(drug1,callback);
},
function(callback) {
getID(drug2,callback);
}
],
function(err, results) {
console.log(results[0]); // id1
console.log(results[1]); // id2
// do the rest..
});

How to handle callbacks in a for loop(Node.JS)

I am trying to write a code with NodeJS where I grab data from an external API and then populate them in MongoDB using Mongoose. In between that, I'll check to see if that particular already exists in Mongo or not. Below is my code.
router.route('/report') // the REST api address
.post(function(req, res) // calling a POST
{
console.log('calling report API');
var object = "report/" + reportID; // related to the API
var parameters = '&limit=100' // related to the API
var url = link + object + apiKey + parameters; // related to the API
var data = "";
https.get(url, function callback(response)
{
response.setEncoding("utf8");
response.on("data", function(chunk)
{
data += chunk.toString() + "";
});
response.on("end", function()
{
var jsonData = JSON.parse(data);
var array = jsonData['results']; // data is return in array of objects. accessing only a particular array
var length = array.length;
console.log(length);
for (var i = 0; i < length; i++)
{
var report = new Report(array.pop()); // Report is the schema model defined.
console.log('^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^');
console.log(i);
console.log('*****************************');
console.log(report);
console.log('*****************************');
// console.log(report['id']);
/*report.save(function(err)
{
if(err)
res.send(err);
});*/
Report.find({id:report['id']}).count(function(err, count) // checks if the id of that specific data already exists in Mongo
{
console.log(count);
console.log('*****************************');
if (count == 0) // if the count = 0, meaning not exist, then only save
{
report.save(function(err)
{
console.log('saved');
if(err)
res.send(err);
});
}
});
};
res.json({
message: 'Grabbed Report'
});
});
response.on("error", console.error);
});
})
My problem is that since NodeJS callbacks are parallel, it is not getting called sequentially. My end result would be something like this :
Calling report API
console.log(length) = 100
^^^^^^^^^^^^^^^^^^^^^^^^
console.log(i) = starts with 0
*******************************
console.log(report) = the data which will be stored inside Mongo
*******************************
number 3 - 7 repeats 100 times as the length is equals to 100
console.log(count) = either 0 or 1
number 9 repeats 100 times
console.log('saved')
number 11 repeats 100 times
Lastly, only the last out of 100 data is stored into Mongo
What I need is some sort of technique or method to handle these callbacks which are executing one after the other and not sequentially following the loop. I am pretty sure this is the problem as my other REST APIs are all working.
I have looked into async methods, promises, recursive functions and a couple others non which I could really understand how to solve this problem. I really hope someone can shed some light into this matter.
Feel free also to correct me if I did any mistakes in the way I'm asking the question. This is my first question posted in StackOverflow.
This problem is termed as the "callback hell".
There's lots of other approaches like using Promise and Async libraries you'll find.
I'm more excited about the native async ES7 will bring,
which you can actually start using today with transpiler library Babel.
But by far the simplest approach I've found is the following:
You take out the long callback functions and define them outside.
router.route('/report') // the REST api address
.post(calling_a_POST)
function calling_a_POST(req, res) {
...
var data = "";
https.get(url, function callback(response) {
...
response.on("end", response_on_end_callback); // --> take out
response.on("error", console.error);
});
}
function response_on_end_callback() { // <-- define here
...
for (var i = 0; i < length; i++) {
var report = new Report(array.pop());
...
Report.find({ id: report['id'] })
.count(Report_find_count_callback); // --> take out
};
res.json({
message: 'Grabbed Report'
});
}
function Report_find_count_callback(err, count) { // <-- define here
...
if (count == 0) {
report.save(function(err) { // !! report is undefined here
console.log('saved');
if (err)
res.send(err); // !! res is undefined here
});
}
}
A caveat is that you won't be able to access all the variables inside what used to be the callback,
because you've taken them out of the scope.
This could be solved with a "dependency injection" wrapper of sorts to pass the required variables.
router.route('/report') // the REST api address
.post(calling_a_POST)
function calling_a_POST(req, res) {
...
var data = "";
https.get(url, function callback(response) {
...
response.on("end", function(err, data){ // take these arguments
response_on_end(err, data, res); // plus the needed variables
});
response.on("error", console.error);
});
}
function response_on_end(err, data, res) { // and pass them to function defined outside
...
for (var i = 0; i < length; i++) {
var report = new Report(array.pop());
...
Report.find({ id: report['id'] })
.count(function(err, count){
Report_find_count(err, count, report, res); // same here
});
};
res.json({ // res is now available
message: 'Grabbed Report'
});
}
function Report_find_count(err, count, report, res) { // same here
...
if (count == 0) {
report.save(function(err) { // report is now available
console.log('saved');
if (err)
res.send(err); // res is now available
});
}
}
When I execute the response_on_end function, I am getting the undefined:1 unexpected token u error.
I am pretty much sure it has something to do with this line: var jsonData = JSON.parse(data)
My response_on_end is as below: var jsonData = JSON.parse(data); // problem here
I realize I made an error here:
function calling_a_POST(req, res) {
...
var data = "";
https.get(url, function callback(response) {
...
//sponse.on("end", function(err, data){
response.on("end", function(err){ // data shouldn't be here
response_on_end(err, data, res);
});
response.on("error", console.error);
});
}
Another problem I could forsee, which actually may not arise here but still would be better to talk about anyways.
The data variable, since it's a string which is a primitive type unlike an object, it is "passed by value".
More info
It's better to wrap the variable in an object and pass the object, because objects in javascript are always "passed by reference".
function calling_a_POST(req, res) {
...
// var data = ""; //
var data_wrapper = {};
data_wrapper.data = {}; // wrap it in an object
https.get(url, function callback(response) {
...
response.on("data", function(chunk){
data_wrapper.data += chunk.toString() + ""; // use the dot notation to reference
});
response.on("end", function(err){
response_on_end(err, data_wrapper, res); // and pass that object
});
response.on("error", console.error);
});
}
function response_on_end_callback(err, data_wrapper, res) {
var data = data_wrapper.data; // later redefine the variable
...
for (var i = 0; i < length; i++) {
var report = new Report(array.pop());
...
You can use async library for controlling your execution flows. And there are also iterators for working with arrays.

How to set a variable to a query? mongodb

How do I set a variable to a query? I am trying to use functions and callbacks in node.js to work through async, but I am not sure how to get a query to equal to a variable. What I am trying to do in this code is take a friend collection that belongs to a user and return the friends result(which I don't think I am doing correctly in the query insertAll) and then find the user's info for each of the query. And then return the results as a render. I am not sure how to call render either with this...
Here is my code:
exports.contactList = function(req, res) {
var insertFriend = function(data, callback) {
var friend = User.findById({_id: user.friendStatus.fuId}, function() {
callback(null, data);
}, friend);
};;
var insertAll = function(coll, callback) {
var queue = coll.slice(0),
friendX;
(function iterate(){
if(queue.length === 0) {
callback();
return;
}
friendX = queue.splice(0,1)[0];
insertFriend(friendX, function(err, friendX) {
if(err) {throw err;}
console.log(friendX + ' inserted');
process.nextTick(iterate);
});
})();
};
insertAll([Friend.findOne({userId: req.signedCookies.userid})], function(){
});
};
A Query object is returned if you do not pass a callback.
From http://mongoosejs.com/docs/queries.html:
When a callback function:
is passed, the operation will be executed immediately with the results passed to the
callback.
is not passed, an instance of Query is returned, which provides a special QueryBuilder
interface for you.

Resources