How to handle node http requests - node.js

I'm trying to understand how to wait for http requests to finish in node. I want to make two http requests and use the results in a function that gets called after the http requests are finished.
I'm using async and request and have been using async.series as following:
var request = require("request");
var express = require('express');
var async = require("async");
app.get('/rx', function(req, res) {
var drug1 = req.query.drug1;
var drug2 = req.query.drug2;
console.log("in rx")
console.log(drug1);
console.log(drug2);
var id1 = '';
var id2 = '';
/*part of code I'm concerned with*/
async.series([
function(callback) {
id1 = getID(drug1);
console.log("function one");
console.log(id1);
callback();
},
function(callback) {
id2 = getID(drug2);
console.log("function two");
console.log(id2);
callback();
}
],
function(err, results) {
console.log(id1);
console.log(id2);
request("http://rxnav.nlm.nih.gov/REST/interaction/interaction.json?list?rxcuis=" + id1 + "&sources=" + id2, function(error, response, body) {
console.log("finished!");
res.json(body);
});
});
});
//returns an int ID
function getID(drugName) {
request("http://rxnav.nlm.nih.gov/REST/Prescribe/rxcui.json?name=" + drugName, function(error, response, body) {
var content = JSON.parse(body);
var id = parseInt(content.idGroup.rxnormId);
console.log("in getID function");
console.log(id);
return id;
});
}
The console output shows:
in rx
advil
ibuprofen
seriesone
undefined
two
undefined
undefined
undefined
finished!
GET /rx?drug1=advil&drug2=ibuprofen 304 345ms
in getID function
153010
in getID function
5640
I want to wait until each http request function is completed, and then proceed to the next portion of code. How do I achieve this?

This question (or variants thereof) has been asked more than 1000 times here on StackOverflow. Therefore I'm not going to explain it but you can search "return async" on this site (the search input at the top right corner) if you want to know more.
The basic problem is that it's impossible to return values from an async function (ever wonder why they accept callbacks?).
For your specific case, you need to change getId() to:
//returns an int ID
function getID(drugName, callback) {
request("http://rxnav.nlm.nih.gov/REST/Prescribe/rxcui.json?name=" + drugName, function(error, response, body) {
var content = JSON.parse(body);
var id = parseInt(content.idGroup.rxnormId);
console.log("in getID function");
console.log(id);
callback(null,id); // <--- this is how you return async values
});
}
Note: the null is because functions in the async.js family expects the first argument to the callback to be an error. So if there are no errors pass null. This by the way is a standard node.js practice.
Then inside async.series() you do:
async.series([
function(callback) {
getID(drug1,callback);
},
function(callback) {
getID(drug2,callback);
}
],
function(err, results) {
console.log(results[0]); // id1
console.log(results[1]); // id2
// do the rest..
});

Related

page renders before getting all the values sorted

I think the rendering takes place before the searching of the string on the files, i have tried different methods but don't seems to get this working. any help will be appreciated. im a noob on to the nodejs. im trying to get the id of the user and query and get all the data and there after see if he is in any of the lists given and finally render the page.
const j = [];
let name = '';
const filename = [];
var ext = '';
module.exports = function(app, express) {
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
app.post('/cusdetails', isLoggedIn, function (req, res) {
var cusid=req.body.cusid;
var insertQuerys = "SELECT * FROM customer WHERE cusid=? ORDER BY rowid DESC LIMIT 1";
connection.query(insertQuerys,[cusid],
function(err, rows){
rows.forEach( (row) => {
name=row.fncus;
});
fs.readdir('./views/iplist', function(err, files) {
if (err)
throw err;
for (var index in files) {
j.push(files[index])
}
j.forEach(function(value) {
var k = require('path').resolve(__dirname, '../views/iplist/',value);
fs.exists(k, function(fileok){
if(fileok) {
fs.readFile(k, function(err, content) {
if (err) throw err;
if (content.indexOf(name) > -1) {
ext = path.extname(k);
filename.push(path.basename(k, ext));
}
});
}
else {
console.log(" FileNotExist ");
}
});
});
});
console.log(filename);
res.render('cusdetails.ejs', {rows: rows, user:req.user , aml: filename });
});
})
You can create simple Promise wrapper and then use it inside async/await function to pause execution until resolved.
// use mysql2 package as it provides promise, less work to write promise wrappers
const mysql = require('mysql2/promise');
// create the connection to database
const connection = mysql.createConnection({
host: 'localhost',
user: 'root',
database: 'test'
});
// sample wrapper
function some(k) {
// more advisable to have local variables, why do you need this to be array?
var filename = [];
return new Promise((resolve, reject) => {
// doing this is also not recommended check nodejs documentation **fs.exists** for more info
fs.exists(k, function(fileok){
if(fileok) {
fs.readFile(k, function(err, content) {
if (err) reject(err);
if (content.indexOf(name) > -1) {
ext = path.extname(k);
filename.push(path.basename(k, ext));
resolve(filename)
}
});
}
else {
// reject(new Error("FileNotExist"))
console.log(" FileNotExist ");
}
});
})
}
// note the use of async
app.post('/cusdetails', isLoggedIn, async function (req, res) {
var cusid=req.body.cusid;
var insertQuerys = "SELECT * FROM customer WHERE cusid=? ORDER BY rowid DESC LIMIT 1";
// using await to pause excution, waits till query is finished
const [rows] = await connection.query(insertQuerys,[cusid])
rows.forEach( (row) => {
name=row.fncus;
});
// then you can
var result = await some(k)
...
Note however this way you loose the advantage of concurrent execution, as it's kindoff blocking. If the result of one call is not used in another, you can execute in parallel and await for result to achieve sequencing like
const [rows] = connection.query(insertQuerys,[cusid])
var result = some(k)
console.log(await rows) // do something
console.log(await result) // do something
JavaScript is asynchronous. This means that if you have a function with a callback (i.e. your query), the callback will be called asynchronously, at an unknown time, while the other code executes.
You need to look up some tutorials how to deal with callbacks, to get a proper understanding of it. Another method is using async/await and/or promises.
Basically, if you take the following code:
console.log("this will print first");
setTimeout(function () {
console.log("this will print last");
}, 1000);
console.log("this will print second");
If you run the code above, the top level is executed synchronously, so, it first calls console.log, then it executes setTimeout, which is synchronous. It sets a timeout, then says "I'm ready", and the code continues to the other console.log. After 1 second (1000 milliseconds), the callback in the setTimeout function is executed, and only then that console.log is called. You can not make the rest of the code wait this way, you need to restructure your code or read into promises.

node - how does async.map work?

After reading about async, I assumed the code below would output to the console the total of all values returned from the http/API call; but it seems to fire immediately after the first http call returns, and only shows a 'total' value equal to the first value returned from the API.
Where is my misunderstanding about how async.map works?
var http = require('https');
const
async = require('async');
var MongoClient = require('mongodb').MongoClient;
var dbUrl = "mongodb://localhost:27017/";
var total = 0;
var tokens = [ {
name : "tron"
}, {
name : 'cardano'
}, {
name : 'nucleus-vision'
}, {
name : 'ripple'
}, {
name : 'litecoin'
}, {
name : 'havven'
}];
function run() {
doStuff();
setInterval(doStuff, 1 * 60 * 1000);
};
function doStuff() {
total = 0;
async.map(tokens, httpGet, function (value){
console.log('async done ', total);
});
}
function httpGet(token, callback) {
var url = 'https://api.coinmarketcap.com/v1/ticker/' + token.name;
http.get( url,
function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
var jsonObj = JSON.parse(body);
var price = parseFloat(jsonObj[0].price);
total += price;
MongoClient.connect(dbUrl, function(err, db) {
if (err)
throw err;
var dbo = db.db("crypto");
dbo.collection("tick").insertOne(jsonObj[0],
function(err, res) {
if (err)
throw err;
db.close();
});
});
callback(price);
});
}).on('error', function(e) {
console.log("Got an error: ", e);
});
};
run();
callback that is passed to an iteratee (httpGet) is used incorrectly. The first argument (price) is considered an error. From the docs:
If iteratee passes an error to its callback, the main callback (for the map function) is immediately called with the error.
So
callback(price);
should rather be
callback(null, price);
So async does not halt after the first iteration.
I believe there are two separate problems here:
As you may know, we cannot use return statements in asynchronous code like we would in synchronous code, which is why we use callbacks instead. Node-style callbacks are on the form function (err, result) {}, where the first parameter is the error (if any) and the second the result of the function (the return value). According to the docs,
Async.map(coll, iteratee, callback) will stop the execution if the
iteratee passes an error to its callback.
As your iteratee-function is calling its callback as such: callback(price), you're effectively stopping execution, as price is passed as the error parameter. What you want to do to "return" the price variable, is to call the callback as so: callback(null, price)
Typically, map-functions are used for
appl[ying] a given function to each element of a list, returning a list of results in the same order.
The map function of the async library does the same, IE: iterates through an array and returns an array of the resulting items, just like the normal map (below) method does.
[1, 2, 3].map(function (nbr) { return nbr*2 }) // returns [2, 4, 6]
The result parameter of your callback (IE, the third parameter to async.map) will be called with an array of prices, and not the summed value of the prices.
async.map(tokens, httpGet, function (error, total) {
console.log(error); // prints undefined (unless there was an error)
console.log(total); // prints an array of prices
});
For summing the values, I would recommend the reduce function, or simply sum the values returned as a result.

Mongoose Async Call Issue with Find Query

I have var movieRecommendation which is being populated from data coming from Mongo DB. Issue is Mongoose Movie.findOne() call is asycn call which is not allowing me to get my final populated movieRecommendation which I need to send back as response.
exports.getRecommendation=function(req,res){
var movieRecommendation = [];
var id=req.params.id;
console.log('----- Get User Recommendation - ' + id);
var url = 'http://52.8.48.113:8080/recommender-server/recommender/v1/recommendations/'+id+'.do';
//make http get request
request({
url: url,
json: true
}, function (error, response, recommendations) {
// res.json(recommendations);
if (!error && response.statusCode === 200) {
recommendations.forEach(function(entry) {
**Movie.findOne({'id':parseInt(entry.itemId)},function(err, movieData){**
entry.movie = movieData;
movieRecommendation.push(entry);
//console.log('rec', movieRecommendation);
console.log(movieRecommendation.length);
});
});
}
console.log("====Final========"+movieRecommendation.length);
//Output = 0
});
res.json(movieRecommendation); // Here movieRecommendation is coming as black Array
};
Please let me know how I can get finally populated movieRecommendation var at end to make it available for response.
For this type of issues we can use Async library. To populate the data finally once all the operations done, we can use async.each collection from Async Library.
For example:
NOTE:Install Async by this command
npm install async to use async library
var async = require("async");
var recomenmendations = [{"data2" : "value2"} , {"data1" : "value2"}, {"data3" : "value3"}, {"data4" : "value4"} ]
var movieRecommendation = [];
async.each(recomenmendations,
function(recomenmendationItem, callback){
console.log("Here you can query the required data using current recomenmendations ITEM");
console.log(recomenmendationItem);
callback();
// Movie.find({'id':parseInt(recomenmendationItem.itemId)},function(err, movieData){
// recomenmendationItem.movie = movieData;
// movieRecommendation.push(entry);
// callback();
// });
},
function(err){
console.log("here you can send your resopnse");
console.log("This section will be executed once all the recomenmendations are processed");
//res.json(movieRecommendation)
}
);
You can query the mongoDB as shown with comment section. You should use callback() once all the operations performed for an iteration.
As I mentioned in one of my comments, use the callback passed to the iterator function and call it inside the Movie.findOne() callback. That way, async.each will know when each step has finished:
async.each(recomendations, function (recomendationItem, callback) {
Movie.findOne({'id':parseInt(entry.itemId)},function(err, movieData){
if (err) return callback(err); // if you have an error on you search, just pass it to the iterator callback
recommendationItem.movie = movieData;
movieRecommendation.push(recommendationItem);
callback();
});
}, function (error) {
if (error) return res.json ({ error: error }); // you should also check if an error ocurred
res.json(movieRecomendation);
});
Just to point out: you can also use async.eachSeries, that will just call the next step of your iteration when the previous one has returned (if that matters to you, but I think it's not your case though) and it has the same signature.
#Vivek Panday replace the following code inside your exports.getRecommendation function to get your expected output. We don't need to use the count variable if we use the callback function. And an important thing is we have to use callback(); once all the process done. I think you have not used callback function properly in The example you have worked out. Use the following code If there is any issue please let me know.
var async = require('async');
var request = require("request");
var movieRecommendation = [];
var id=req.params.id;
console.log('----- Get User Recommendation - ' + id);
var url = 'http://52.8.48.113:8080/recommender-server/recommender/v1/recommendations/'+id+'.do';
//make http get request
request({
url: url,
json: true
}, function (error, response, recommendations) {
if (!error && response.statusCode === 200) {
console.log('recommendation lenght '+ recommendations.length);
async.each(recommendations,
function(recommendationItem, callback){
Movie.findOne({'id':parseInt(recommendationItem.itemId)},function(err, movieData){
recommendationItem.movie = movieData;
movieRecommendation.push(recommendationItem);
//you have to use callback(); once all your process is done
callback();
});
},
function(err){
//you should use this function, this will be execute once all the process done
console.log(movieRecommendation);
console.log("finally callback");
res.json(movieRecommendation);
}
);
}
});
I have tried as per given suggestion above ..
var async = require("async");
var recomenmendations = [{"data2" : "value2"} , {"data1" : "value2"}, {"data3" : "value3"}, {"data4" : "value4"} ]
var movieRecommendation = [];
async.each(recomenmendations,
function(recomenmendationItem, callback){
console.log("Here you can query the required data using current recomenmendations ITEM");
console.log(recomenmendationItem);
// Movie.find({'id':parseInt(recomenmendationItem.itemId)},function(err, movieData){
recomenmendationItem.movie = movieData;
movieRecommendation.push(entry);
console.log("any data"); // line y
});
callback();
},
function(err){
console.log("here you can send your resopnse"); // line x
console.log("This section will be executed once all the
recomenmendations are processed");
//res.json(movieRecommendation)
}
);
But still face same issue line x is printing before line y ,which is making again same issue.
However I have tried something given below and achieved expected result .
exports.getRecommendation=function(req,res){
var movieRecommendation = [];
var id=req.params.id;
console.log('----- Get User Recommendation - ' + id);
var url = 'http://52.8.48.113:8080/recommender-server/recommender/v1/recommendations/'+id+'.do';
//make http get request
request({
url: url,
json: true
}, function (error, response, recommendations) {
// res.json(recommendations);
if (!error && response.statusCode === 200) {
console.log('recommendation lenght '+ recommendations.length);
// recommendations.forEach(function(entry) {
var count=0;
async.each(recommendations,function(recommendationItem){
// console.log(recommendationItem);
Movie.findOne({'id':parseInt(recommendationItem.itemId)},function(err, movieData){
recommendationItem.movie = movieData;
movieRecommendation.push(recommendationItem);
count ++;
console.log('final res length : ' + movieRecommendation.length);
console.log('final res length count : ' + count +' and item recomm lenght ' + recommendations.length );
if(count === recommendations.length){
console.log(' =====Final=====> here you can send your response =========' + movieRecommendation.length);
res.json(movieRecommendation);
}
});
// callback();
});
}
});
};
Still I am open for any feedback and suggestions.

How to set a variable to a query? mongodb

How do I set a variable to a query? I am trying to use functions and callbacks in node.js to work through async, but I am not sure how to get a query to equal to a variable. What I am trying to do in this code is take a friend collection that belongs to a user and return the friends result(which I don't think I am doing correctly in the query insertAll) and then find the user's info for each of the query. And then return the results as a render. I am not sure how to call render either with this...
Here is my code:
exports.contactList = function(req, res) {
var insertFriend = function(data, callback) {
var friend = User.findById({_id: user.friendStatus.fuId}, function() {
callback(null, data);
}, friend);
};;
var insertAll = function(coll, callback) {
var queue = coll.slice(0),
friendX;
(function iterate(){
if(queue.length === 0) {
callback();
return;
}
friendX = queue.splice(0,1)[0];
insertFriend(friendX, function(err, friendX) {
if(err) {throw err;}
console.log(friendX + ' inserted');
process.nextTick(iterate);
});
})();
};
insertAll([Friend.findOne({userId: req.signedCookies.userid})], function(){
});
};
A Query object is returned if you do not pass a callback.
From http://mongoosejs.com/docs/queries.html:
When a callback function:
is passed, the operation will be executed immediately with the results passed to the
callback.
is not passed, an instance of Query is returned, which provides a special QueryBuilder
interface for you.

Call .post() API-Easy

I'm using API-easy , how do I get this result in _id_user and send the call to .post() thanks. e.g
var APIeasy = require('api-easy'),
assert = require('assert');
var _id_user;
var suite = APIeasy.describe('Test User');
suite.use('localhost', 3000)
.discuss('Test')
.setHeader('Content-Type', 'application/x-www-form-urlencoded')
.post('/user/authenticate', {data: '{"email":"emailuser#email.com","password":"123456"}')
.expect('should respond with ID user', function (err, res, body) {
_id_user = body; // I need this result to be sent in the next call .post()
}).next()
.post('/user/validate',{ data : _id_user}) // this result always comes null
.expect('should respond TRUE', function (_err, _res, _body) {
}).export(module);
The right way of dealing with this is to use the before() call to modify your post parameters. You can directly modify the contents of the 'outgoing' request.
var APIeasy = require('api-easy'),
assert = require('assert');
var suite = APIeasy.describe('Test User');
suite.use('localhost', 3000)
.discuss('Test')
.setHeader('Content-Type', 'application/x-www-form-urlencoded')
.post('/user/authenticate', {data: '{"email":"emailuser#email.com","password":"123456"}')
.expect('should respond with ID user', function (err, res, body) {
suite.before('setUserId', function(outgoing) {
//use outgoing.body for post requests and outgoing.uri for get requests
outgoing.body = outgoing.body.replace('_ID_USER',body);
return outgoing;
});
}).next()
.post('/user/validate',{ data : '_ID_USER'})
.expect('should respond TRUE', function (_err, _res, _body) {
//you can unbefore() here if you need it
suite.unbefore('setUserId');
}).export(module);
That's because of asynchronous nature of callback function. Do the second post inside the callback function, i.e. after _id_user=body;

Resources