So, here is the problem. I found how to do http request in Node.js, so that I could download and parse remote JSON file. It all works fine but nothing happens after that. I have a function and in that there is if condition that doesn't get executed. It simply cannot get past the http request. Is there something I am missing?
var remoteStamp;
if (typeof force == "undefined") {
var timeurl = "http://" + parsedconfig.weburl + "/timestamp.json";
request(timeurl, { json: true }, (err, res, body) => {
if (err) { return console.log(err); }
console.log(body.timestamp);
remoteStamp = body.timestamp;
});
}
if (remoteStamp < parsedconfig.timestamp || force == "local") {
//something should happen here, all the values check out - still nothing happens
}
you are using callback so all the code which you want to execute after request is completed should be inside the callback
var remoteStamp;
if (typeof force == "undefined") {
var timeurl = "http://" + parsedconfig.weburl + "/timestamp.json";
request(timeurl, { json: true }, (err, res, body) => {
if (err) { return console.log(err); }
console.log(body.timestamp);
remoteStamp = body.timestamp;
if (remoteStamp < parsedconfig.timestamp || force == "local") {
//something should happen here, all the values check out - still nothing happens
}
});
}
Or you can use request-promise library to do this in promises. https://github.com/request/request-promise
On how to use promises: https://developers.google.com/web/fundamentals/primers/promises
Related
I have the following code:
function get_status(){
try {
/* GET - status */
async.map(['http://url1.com/', 'http://url2.com/', 'http://url3.com/'], function(value, callback) {
/* GET - request status */
request.post({
url: value,
form: { 'mycustomdata': ""+mycustomdata+"" },
method: 'POST'}, function(err, res, body) {
/* CHECK - response */
if (!err && typeof body !== 'undefined' && res.statusCode == 200) {
console.log('get status success...')
callback();
} else {
callback('failed to get status');
}
})
}, function(err, results) {
if (err) {
console.log(err); return false;
} else {
console.log('finished...') }
})
}
I need when all three urls in async.map processing is finished(some url respond some not) to call where it outputs 'finished...' and from this output to callthis function again...so that function runs every time all url is processed.
So in short..i need when async.map process all urls to output to console 'finished...' and run same function again.
But i m getting only from console...
get status success...
failed to get status..
get status sucess..
i never get called
'finished'...
so that i can cal function itsef...could you please light me how it needs to be written?
You're only ever passing in the err param to the final callback. I believe, from my last use of async.js, you need to pass in null as the first param to continue to the next function, here being the final function. Otherwise is skips to the final function as an error.
callback(null, results);
If you want it to call itself again, that's just simple recursion. Wrap the async.map in a function, and when console.log('finished...') runs, call the function again.
Ok i made full example so that you can see what is problem and you can try to change code:
var async = require('async'),
request = require('request');
var myUrls = [ 'http://56.123.65.86:8080/api/server_status/',
'http://88.96.42.122:8080/api/server_status/',
'http://12.23.32.25:8080/api/server_status/',
'http://251.214.44.58:8080/api/server_status/',
'http://122.23.32.54:8080/api/server_status/' ]
async.map(myUrls, function(url, callback) {
console.log('getting url...'+url)
request(url, function(error, response, html) {
// Some processing is happening here before the callback is invoked
if (typeof response !== undefined) {
console.log('response ok...'+url)
return callback(null, html)
} else {
console.log('response failed...'+url)
return callback(error, html);
}
});
}, function(err, results) {
if (results) {
console.log('all finished...')
}
});
I im getting this:
getting url...http://56.123.65.86:8080/api/server_status/
getting url...http://88.96.42.122:8080/api/server_status/
getting url...http://12.23.32.25:8080/api/server_status/
getting url...http://251.214.44.58:8080/api/server_status/
getting url...http://122.23.32.54:8080/api/server_status/
And need to get this:
getting url...http://56.123.65.86:8080/api/server_status/
getting url...http://88.96.42.122:8080/api/server_status/
getting url...http://12.23.32.25:8080/api/server_status/
getting url...http://251.214.44.58:8080/api/server_status/
getting url...http://122.23.32.54:8080/api/server_status/
all finished...
I cannot get all finished...so it stays with getting url and that stays forever...so i cannot get idea why is all finished... not triggeret after all items are processed...
I have a post method, whose request input is a list of newLink objects (newLink with attributes linkUrl and status)
I am using async.map for iterating over my URLs to check if the links are active or not.
newLinks contains links like {www.google.com,www.nourl.com,www.xyz.com} I am expecting like after all the request are processed and setting corresponding status as true or false, I want this to send using res.send(newLinks)
But the console is giving the below results: "www.google.com is up", then calling res.send(), then executing "www.nourl.com is up" and "www.xyz.com is up"
So basically here, after the first url request , my code below is executing the function outside the async loop. I thought async will only allow the next piece of code to execute only after all the urls are validated.
app.post('/myposturl', function(req , res){
var request = require('request');
let linkDetails= req.body.linkDetails;
var i = 0;
async.map(linkDetails, function(newLink, callback) {
var Url = "url";
var url = newLink.linkUrl;
var proxiedRequest = request.defaults({'proxy': Url});
proxiedRequest(url , function (error, response, body) {
if(error){
console.log('Err: '+ error);
}
if (!error) {
if(response.statusCode == 200 || response.statusCode == 201 ||
response.statusCode == 202){
console.log(url + ' is up!!');
newLink.isActive = true;
}
if(response.statusCode == 301 || response.statusCode == 302){
console.log(url + ' is redirecting us!!');
return false;
}
}
});
callback();
} , function(err, linkDetails) {
res.send(linkDetails);
});
//tried res.send here as well.
});
}
The callback of async.map should invoke inside proxiedRequest. What your code is doing now: invoke callback immediately before the proxiedRequest finished. Also return false; does not work in asynchronous function. You should return the new status like this callback(null, newLink). After all the request are processed, the newLinkDetails will be the array of all newLink.
Note, that since this function applies the iteratee to each item in parallel, there is no guarantee that the iteratee functions will complete in order.
If you need to keep the order, user mapSeries insted.
Please read the doc of async.map for more. Hope it helps.
app.post('/myposturl', function(req , res){
//other codes
async.map(linkDetails, function(newLink, callback) {
//other codes
proxiedRequest(url , function (error, response, body) {
if(error){
console.log('Err: '+ error);
callback(error);
//^ ^ ^ ^ ^
// Validation failed, return from here
}
else {
//some validation & set newLink.isActive
callback(null, newLink);
// ^ ^ ^ ^ ^
//return newLink status by invoking the callback
}
});
}, function(err, newLinkDetails) {
//err = if any Validation failed
// now all the request are processed,newLinkDetails is array of all newLink's
res.send(newLinkDetails);
});
});
Usually when using async.js, I follow these two principles:
Always call the callback at least AND at most once during your async function.
Call the callback only when the async function is complete OR if an error has occurred. If the latter occurs, call the callback passing the error AND stop further execution of the async function e.g. return callback(error)
I would revise your code as below:
var request = require('request');
app.post('/myposturl', function (req , res) {
async.mapSeries(req.body.linkDetails || [], function(newLink, callback) {
var Url = "url";
var proxiedRequest = request.defaults({ 'proxy': Url });
proxiedRequest(newLink.linkUrl, function (err, response, body) {
if (err)
return callback(err);
// I'm assuming you don't want to stop checking the links for bad status codes
if ([301, 302].indexOf(response.statusCode) > -1){
return callback(null, url + ' is redirecting us!!');
if ([200, 201, 202].indexOf(response.statusCode) == -1) {
return callback(null, url + ' came back with ' + response.statusCode);
console.log(url + ' is up!!');
newLink.isActive = true;
callback(null, newLink);
});
}, function (err, linkDetails) {
// when all links get checked, it will come down here
// or if an error occurs during the iteration, it will come down here
console.log(err, linkdetails);
res.send(linkDetails);
});
});
If you only want to get back active links, you may also want to check out async.filterSeries(). Here the callback would need to be passed a boolean in its second argument.
I am using Facebook Graph NodeJS API to fetch user_posts. The response has pagination and therefore I need to loop over the response to fetch all the posts. I am using following route to fetch facebook posts and I am looping over pagination using get_user_statuses function:
var posts = "";
function get_user_statuses(response_posts, res) {
var link_regex = /https?:\/\/(?:www\.|(?!www))[^\s\.]+\.[^\s]{2,}|www\.[^\s]+\.[^\s]{2,}/g;
var isNextPageAvailable = true;
if ("paging" in response_posts) {
var nextPage = response_posts.paging.next;
isNextPageAvailable = true;
} else {
isNextPageAvailable = false;
}
for (var i = 0; i < response_posts.data.length; i++) {
var post = response_posts.data[i].message + " ";
if ("message" in response_posts.data[i]) {
posts += post.replace(link_regex, "");
}
}
if (nextPage !== undefined) {
request(nextPage, function (error, response, body) {
if (!error && response.statusCode === 200) {
get_user_statuses(JSON.parse(body));
} else {
console.log(error);
}
});
}
if (!isNextPageAvailable){
//Sending posts to facebook Modal
console.log(posts);
res.send(JSON.stringify({posts: posts})); //res is not defined here
}
}
router.post('/fbData', function (req, response, next) {
FB.setAccessToken(req.body.access_token);
FB.api('/me?fields=posts&posts.limit=1000', function (res) {
if (!res || res.error) {
if (!res) {
response.send(JSON.stringify({error: 'An error occurred. Please try again.'}))
}
response.send(JSON.stringify({error: response.error}));
return;
}
get_user_statuses(res.posts, response); //Passing response object here
});
});
The issue is that response object passed from express route is not defined in get_user_statuses function. Now I have two question:
Why is response object is not defined?
Is there better approach to achieve this arrangement?
res is not defined because you forgot to pass it in internal call. Change get_user_statuses(JSON.parse(body)); to get_user_statuses(JSON.parse(body), res); and it should work
I solved my problem. I needed to create a function with a callback. In case anyone else is stuck at this kind of issue, this post helped me resolve it:
[How to recurse asynchronously over API callbacks in node.js?
I'm using cheerio, request and Node.js.
When I run the script below, it outputs names in a wrong order. I believe that it's caused by asynchronous nature of it, how can I make it work in the "right" order? Do I need to use a sync package or is there a way to change it in a way so it'll work in a sync way?
app.get('/returned', function (req, res) {
for (var y = 0; y < 10; y++) {
var url = "http://example.com" + y + "/person.html";
request(url, function (err, resp, body) {
$ = cheerio.load(body);
var links = $('#container');
var name = links.find('span[itemprop="name"]').html(); // name
if (name == null) {
console.log("returned null");
} else {
console.log(name);
}
});
}
});
Promise makes this relatively easy:
app.get('/returned', function (req, res) {
let urls = [];
for (let y = 0; y < 10; y++) {
urls.push('http://example.com' + y + '/person.html');
}
Promise.all(urls.map(function (url) {
return new Promise(resolve, reject) {
request(url, function (err, resp, body) {
if (err) {return reject(err);}
let $ = cheerio.load(body);
let links = $('#container');
let name = links.find('span[itemprop="name"]').html(); // name
resolve({name: name, links: links, url: url});
});
});
}).then(function (result) {
result.forEach(function (obj) {
if (obj.name == null) {
console.log(obj.url, "returned null");
} else {
console.log(obj.url, obj.name);
}
});
}).catch(function (err) {
console.log(err);
});
});
I started by creating an array of urls to get, then I mapped that to an array of promises. When each of the requests are complete, i resolved the promise with the name, url, and links. When all promises were complete, I then looped over the result which will will be in the original order. This runs in parallel.
Nope, you shouldn't have to use a sync package. IMO the cleanest way is to use a mature 3rd party library.
I'd recommend async.
The async.series method would execute all request functions in the order they are given, then allow you to register a callback to fire when all requests have been made, or when an error has occurred.
https://github.com/caolan/async#seriestasks-callback
I have var movieRecommendation which is being populated from data coming from Mongo DB. Issue is Mongoose Movie.findOne() call is asycn call which is not allowing me to get my final populated movieRecommendation which I need to send back as response.
exports.getRecommendation=function(req,res){
var movieRecommendation = [];
var id=req.params.id;
console.log('----- Get User Recommendation - ' + id);
var url = 'http://52.8.48.113:8080/recommender-server/recommender/v1/recommendations/'+id+'.do';
//make http get request
request({
url: url,
json: true
}, function (error, response, recommendations) {
// res.json(recommendations);
if (!error && response.statusCode === 200) {
recommendations.forEach(function(entry) {
**Movie.findOne({'id':parseInt(entry.itemId)},function(err, movieData){**
entry.movie = movieData;
movieRecommendation.push(entry);
//console.log('rec', movieRecommendation);
console.log(movieRecommendation.length);
});
});
}
console.log("====Final========"+movieRecommendation.length);
//Output = 0
});
res.json(movieRecommendation); // Here movieRecommendation is coming as black Array
};
Please let me know how I can get finally populated movieRecommendation var at end to make it available for response.
For this type of issues we can use Async library. To populate the data finally once all the operations done, we can use async.each collection from Async Library.
For example:
NOTE:Install Async by this command
npm install async to use async library
var async = require("async");
var recomenmendations = [{"data2" : "value2"} , {"data1" : "value2"}, {"data3" : "value3"}, {"data4" : "value4"} ]
var movieRecommendation = [];
async.each(recomenmendations,
function(recomenmendationItem, callback){
console.log("Here you can query the required data using current recomenmendations ITEM");
console.log(recomenmendationItem);
callback();
// Movie.find({'id':parseInt(recomenmendationItem.itemId)},function(err, movieData){
// recomenmendationItem.movie = movieData;
// movieRecommendation.push(entry);
// callback();
// });
},
function(err){
console.log("here you can send your resopnse");
console.log("This section will be executed once all the recomenmendations are processed");
//res.json(movieRecommendation)
}
);
You can query the mongoDB as shown with comment section. You should use callback() once all the operations performed for an iteration.
As I mentioned in one of my comments, use the callback passed to the iterator function and call it inside the Movie.findOne() callback. That way, async.each will know when each step has finished:
async.each(recomendations, function (recomendationItem, callback) {
Movie.findOne({'id':parseInt(entry.itemId)},function(err, movieData){
if (err) return callback(err); // if you have an error on you search, just pass it to the iterator callback
recommendationItem.movie = movieData;
movieRecommendation.push(recommendationItem);
callback();
});
}, function (error) {
if (error) return res.json ({ error: error }); // you should also check if an error ocurred
res.json(movieRecomendation);
});
Just to point out: you can also use async.eachSeries, that will just call the next step of your iteration when the previous one has returned (if that matters to you, but I think it's not your case though) and it has the same signature.
#Vivek Panday replace the following code inside your exports.getRecommendation function to get your expected output. We don't need to use the count variable if we use the callback function. And an important thing is we have to use callback(); once all the process done. I think you have not used callback function properly in The example you have worked out. Use the following code If there is any issue please let me know.
var async = require('async');
var request = require("request");
var movieRecommendation = [];
var id=req.params.id;
console.log('----- Get User Recommendation - ' + id);
var url = 'http://52.8.48.113:8080/recommender-server/recommender/v1/recommendations/'+id+'.do';
//make http get request
request({
url: url,
json: true
}, function (error, response, recommendations) {
if (!error && response.statusCode === 200) {
console.log('recommendation lenght '+ recommendations.length);
async.each(recommendations,
function(recommendationItem, callback){
Movie.findOne({'id':parseInt(recommendationItem.itemId)},function(err, movieData){
recommendationItem.movie = movieData;
movieRecommendation.push(recommendationItem);
//you have to use callback(); once all your process is done
callback();
});
},
function(err){
//you should use this function, this will be execute once all the process done
console.log(movieRecommendation);
console.log("finally callback");
res.json(movieRecommendation);
}
);
}
});
I have tried as per given suggestion above ..
var async = require("async");
var recomenmendations = [{"data2" : "value2"} , {"data1" : "value2"}, {"data3" : "value3"}, {"data4" : "value4"} ]
var movieRecommendation = [];
async.each(recomenmendations,
function(recomenmendationItem, callback){
console.log("Here you can query the required data using current recomenmendations ITEM");
console.log(recomenmendationItem);
// Movie.find({'id':parseInt(recomenmendationItem.itemId)},function(err, movieData){
recomenmendationItem.movie = movieData;
movieRecommendation.push(entry);
console.log("any data"); // line y
});
callback();
},
function(err){
console.log("here you can send your resopnse"); // line x
console.log("This section will be executed once all the
recomenmendations are processed");
//res.json(movieRecommendation)
}
);
But still face same issue line x is printing before line y ,which is making again same issue.
However I have tried something given below and achieved expected result .
exports.getRecommendation=function(req,res){
var movieRecommendation = [];
var id=req.params.id;
console.log('----- Get User Recommendation - ' + id);
var url = 'http://52.8.48.113:8080/recommender-server/recommender/v1/recommendations/'+id+'.do';
//make http get request
request({
url: url,
json: true
}, function (error, response, recommendations) {
// res.json(recommendations);
if (!error && response.statusCode === 200) {
console.log('recommendation lenght '+ recommendations.length);
// recommendations.forEach(function(entry) {
var count=0;
async.each(recommendations,function(recommendationItem){
// console.log(recommendationItem);
Movie.findOne({'id':parseInt(recommendationItem.itemId)},function(err, movieData){
recommendationItem.movie = movieData;
movieRecommendation.push(recommendationItem);
count ++;
console.log('final res length : ' + movieRecommendation.length);
console.log('final res length count : ' + count +' and item recomm lenght ' + recommendations.length );
if(count === recommendations.length){
console.log(' =====Final=====> here you can send your response =========' + movieRecommendation.length);
res.json(movieRecommendation);
}
});
// callback();
});
}
});
};
Still I am open for any feedback and suggestions.