I am building a nodejs application.
Location.find({} ,function (err, result){
var locations = [];
result.forEach(function(listItem, i){
url = "https://maps.googleapis.com/maps/api/geocode/json?address=" + encodeURIComponent(listItem.address) + "&key=AIzaSyDQsX4Kci3xHp1xy6ZjT-5lsLNI-J-CH-8";
request(url, function(error, response, body) {
all = JSON.parse(body);
locations.push({des: result[i].placeinfo,lat: all.results[0].geometry.location.lat, lng: all.results[0].geometry.location.lng });
if (i == result.length - 1) {
res.render("index.ejs", { layout: false,locationmap:locations});
}
});
});
});
I have two problems here.
My loop runs 4 times, when i try to console.log() the i var its shows 4 time in the console.
why cant i use the request body outside of the loop i did some workaround and did the res.render inside the if statement.
You may like to use asyncjs eachSeries function but you need to install(npm install asyncjs --save) and then use like
var async = require('asyncjs');
Location.find({} ,function (err, result){
var locations = [];
async.eachSeries(result, function iteratee(listItem, callback) {
url = "https://maps.googleapis.com/maps/api/geocode/json?address=" + encodeURIComponent(listItem.address) + "&key=AIzaSyDQsX4Kci3xHp1xy6ZjT-5lsLNI-J-CH-8";
request(url, function(error, response, body) {
all = JSON.parse(body);
locations.push({des: result[i].placeinfo,lat: all.results[0].geometry.location.lat, lng: all.results[0].geometry.location.lng });
callback(error, body);
});
}, function done() {
res.render("index.ejs", { layout: false,locationmap:locations});
});
});
Related
I have a code snippet like the following.
async.parallel({
foo: function(call) {
request.get('...?node=foo', function (error, response, body) {
var last=JSON.parse(body);
call(null,last.data.result);
});
},
bar: function(call) {
request.get('...node=bar', function (error, response, body) {
var last=JSON.parse(body);
call(null,last.data.result);
});
}
}, function(err, results) {
callback(results)
});
but I want to that in loop...that is to say,like the following.
var node = ['foo', 'bar'];
async.parallel({
for (var i = 0; i < node.length; i++) {
node[i]: function(call) {
request.get('...?node=' + node[i] + '', function(error, response, body) {
var last = JSON.parse(body);
call(null, last.data.result);
});
}
}
}, function(err, results) {
callback(results)
});
it does not work...how to make it?
can you help me?
You have one of two options. One is to read the documentation for async.map(), which would be the more canonical approach using async.
That would look something like this:
async.map(['foo', 'bar'], function (node, callback) {
request.get('...?node=' + encodeURIComponent(node), function (error, response, body) {
if (error) return callback(error)
var last = JSON.parse(body)
callback(null, last.data.results)
})
}, function (error, results) {
// handle error
callback(results)
})
Your second option would be to use Promise.all() with a synchronous Array#map(), removing the dependency for async entirely, though you'll probably want a polyfill like bluebird until Promises are used in more substantial percentage of browsers.
Here's how you'd implement the latter solution using native Promises:
Promise.all(['foo', 'bar'].map(function (node) {
return new Promise(function (resolve, reject) {
request.get('...?node=' + encodeURIComponent(node), function (error, response, body) {
if (error) return reject(error)
var last = JSON.parse(body)
resolve(last.data.result)
})
})
})).then(function (results) {
callback(results)
}).catch(function (error) {
// handle error
})
I want to request multiple pages with request in async.forEach. I have several links, and I want to go to each link and get the sublinks from it. Then I want to evaluate each sublink and push results to dataarray. But only the first one sublink is evaluated multiple times, for each links array element. But I want to evaluate each sublink. Here is my code:
var dataarray = [];
var links = ["www.mysite.com/1","www.mysite.com/2","www.mysite.com/3"];
async.whilst(
function() { console.log("Number of links left: " + links.length); return links.length > 0; },
function(innerCallback){
var i1 = getRandomInt(1, links.length); //getRandomInt is a function that gives a random integer in array.
var Theurl = links[i1-1];
request({
url: Theurl,
//More paramaters to request here...
}, function(error, response, body) {
if (response) {
//Evaluate some jquery to get a variable allsublinks used later on.
//So I get several sub-links in each element in links array.
var allsublinks = stuff;
//Now I define a function to be used in async.forEach. I basically need to
//loop over allsublinks array that contains different url links.
function getInfo(name, thiscallback) {
console.log("Online");
console.log("Current url: "+name);
request({
url: name,
//Other parameters to request.
}, function(error, response, body) {
if (response) {
//Here I evaluate some jquery again to get the datafromthislink variable
dataarray.push(datafromthislink);
setTimeout(function() { thiscallback(); });
}
})
}
//Now I use async.forEach to loop over allstorelinks.
async.forEach(allsublinks, getInfo, function(err, results) {
console.log("dataarray: ",dataarray);
setTimeout(function() { links.splice(i1-1, 1); innerCallback(); });
});
}
})
},
function(err){
console.log("All Done");
})
What am I doing wrong?
Best Regards
This could be the cleaner version of what you are trying to achieve, which clear your concept of using callbacks properly.
var dataarray = [];
var links = ["www.mysite.com/1", "www.mysite.com/2", "www.mysite.com/3"];
//Now I define a function to be used in async.forEach. I basically need to
//loop over allsublinks array that contains different url links.
function getInfo(link, infoCallback) {
console.log("Online");
console.log("Current url: " + link);
request({
url: link,
//Other parameters to request.
}, function (error, response, body) {
if (error) console.log(error);
if (response) {
//Here I evaluate some jquery again to get the datafromthislink variable
dataarray.push(body.datafromthislink);
}
infoCallback();
})
}
function getLink(link, linkCallback) {
console.log('checking link: ' + link);
request({
url: link,
//More paramaters to request here...
}, function (error, response, body) {
if (error) console.log(error);
if (response) {
//Evaluate some jquery to get a variable allsublinks used later on.
//So I get several sub-links in each element in links array.
var allsublinks = body.stuff;
//Now I use async.forEach to loop over allstorelinks.
async.forEach(allsublinks, getInfo, linkCallback);
} else {
linkCallback();
}
})
}
async.each(links, getLink, function (err) {
if (error) console.log(error);
console.log("All Done");
console.log(dataarray);
});
The code I wrote so far is as below.
I don't need the whole response but just part of it.
var request = require('request');
var async = require('async');
var asyncTasks = [];
var install;
async.series([
function (callback) {
setTimeout(function () {
request('URL', function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body); // Show the HTML for the Google homepage.
}
});
}, 5000);
},
function (callback) {
setTimeout(function () {
request('URL', function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body); // Show the HTML for the Google homepage.
}
});
}, 5000);
}
],
function (error, results) {
console.log(results);
});
One approach to do the above concurrently would be to use async.parallel - of the form of:
async.parallel([
function(){ ... },
function(){ ... }
], callback);
Another approach is to use a Promises library - BlueBird or Q are good choices.
Q.All is of the form of:
return Q.all([
promise1,
promise2
]).spread(function (resultFromPromise1, resultFromPromise2) {
// do something with the results...
});
You could use one of these approaches to parallelise the two calls. The outputs of each will give you an array containing the results of each call respectively.
Here is a simple illustration of each approach:
Using Async.js
var async = require('async');
var task = function (cb, count) {
setTimeout(function () {
cb(null, "complete: " + count);
}, 1000);
};
async.parallel([
function (cb) {
task(cb, 'one');
},
function (cb) {
task(cb, 'two');
}
], function (err, results) {
console.log(results);
//[ 'complete: one', 'complete: two' ]
});
Using Q:
var Q = require('q');
function task1(cb, count) {
var deferred = Q.defer();
setTimeout(function () {
return deferred.resolve(cb(null, count));
}, 1000);
return deferred.promise;
}
var myCb = function (err, count) {
return "complete: " + count;
};
Q.all([task1(myCb, 'one'), task1(myCb, 'two')])
.then(function (results) {
console.log(results);
//[ 'complete: one', 'complete: two' ]
});
Let me know if you are unclear.
Promises are there to help you out in such a case.
I would prefer to use 'Q' library.
I have modified your code to use Q library
var Q = require('q');
var request = require('request');
function makeCall() {
Q.all([req1(), req2()])
.spread(function (res1, res2) {
// This block is executed once all the functions( Provided in Q.all() ) are finished its execution.
// Use responses from called functions
}, function (err) {
// Error, If any
});
}
function req1() {
var defer = Q.defer();
var url = ''; // Specify URL
var options = {
method: 'get', // Method to use
url: url
}
request(options, function (err, res, body) {
if (err) {
return defer.reject(err);
}
return defer.resolve(body);
});
return defer.promise;
}
function req2() {
var defer = Q.defer();
var url = ''; // Specify URL
var options = {
method: 'get', // Method to use
url: url
}
request(options, function (err, res, body) {
if (err) {
return defer.reject(err);
}
return defer.resolve(body);
});
return defer.promise;
}
You can find docs for Q library here : Q docs
For a certain route, I have the following code:
router.get('/:id', function(req, res) {
var db = req.db;
var matches = db.get('matches');
var id = req.params.id;
matches.find({id: id}, function(err, obj){
if(!err) {
if(obj.length === 0) {
var games = Q.fcall(GetGames()).then(function(g) {
console.log("async back");
res.send(g);
}
, function(error) {
res.send(error);
});
}
...
});
The function GetGames is defined as follows:
function GetGames() {
var url= "my-url";
request(url, function(error, response, body) {
if(!error) {
console.log("Returned with code "+ response.statusCode);
return new Q(body);
}
});
}
I'm using the request module to send a HTTP GET request to my URL with appropriate parameter, etc.
When I load /:id, I see "Returned with code 200" logged, but "async back" is not logged. I'm also not sure that the response is being sent.
Once GetGames returns something, I want to be able to use that returned object in the route for /:id. Where am I going wrong?
Since GetGames is an async function write it in node.js callback pattern:
function GetGames(callback) {
var url= "my-url";
request(url, function(error, response, body) {
if(!error) {
console.log("Returned with code "+ response.statusCode);
return callback(null,body)
}
return callback(error,body)
});
}
Then use Q.nfcall to call the above function and get back a promise:
Q.nfcall(GetGames).then(function(g) {
})
.catch()
While building a fairly complex scraper i stumbled upon a problem with a control flow of my code.
What's going on in code below:
1) request a URL
2) scrape NEWURL from the results
3) pass it to readability API as first async function
4) here comes the trouble — i never get the next async function which saves readabilityData to DB
How to solve this problem?
I'm new to JS, so please feel free to point out at any issues with my code.
request(URL, function(error, response, html) {
if (!error) {
var $ = cheerio.load(html);
NEWURL = data.find('a').attr('href');
readabilityData = {}
var articleUrl = 'https://readability.com/api/content/v1/parser?url=' + NEWURL + token;
async.series([
function(){
request(articleUrl, function(error, response, html) {
if (!error) {
readabilityData = response.toJSON();
}
});
},
function(readabilityData){
Article.findOne({
"link": url // here's the
}, function(err, link){
if(link) {
console.log(link)
} else {
var newArticle = new Article({
// write stuff to DB
});
newArticle.save(function (err, data) {
// save it
});
}
});
}
],
function(err){
console.log('all good — data written')
});
});
}
});
You need to call the callback parameter that's passed into the functions of the async.series call when each function's work is complete. That's how async.series knows that it can proceed to the next function. And don't redefine readabilityData as a function parameter when you're trying to use it to share data across the functions.
So something like:
var readabilityData = {};
async.series([
function(callback){
request(articleUrl, function(error, response, html) {
if (!error) {
readabilityData = response.toJSON();
}
callback(error);
});
},
function(callback){
Article.findOne({
"link": url // here's the
}, function(err, link){
if(link) {
console.log(link);
callback();
} else {
var newArticle = new Article({
// write stuff to DB
});
newArticle.save(function (err, data) {
// save it
callback(err);
});
}
});
}
],
function(err){
console.log('all good — data written')
});