What download images from remote url - node.js

I've array of url images. I need download asynchronous this images from remote url to my server.
EXAMPLE:
// i need this function return array localfiles after download
function someFunctionAsDownloadedRemoteIMages(arrayOfRemoteUrls){
localImages = [];
arrayOfRemoteUrls.forEach(function(url){
request.head(url, function(err, res, body){
newImageName = randomInt(100000, 999999);
var filename = 'catalog/import/'+newImageName+'.jpg';
request(url, {encoding: 'binary'}, function(error, response, body) {
fs.writeFile('image/'+filename, body, 'binary', function (err) {
if(err)
return;
localImages.push(filename);
});
});
});
});
}
var remoteImagesArray = ["http://example.com/1.jpg", "http://example.com/1444.jpg","http://example.com/ddsgggg.jpg"];
localImagesArray = someFunctionAsDownloadedRemoteIMages(remoteImagesArray);
someFunctionProccess(localImagesArray);

If you want it to asynchronously return anything you must use a callback pattern instead of returning a value from a function. With that said you also need a way for the final result callback to fire once all images have been loaded. I would suggest using a module like async and use the map function it provides. The map function will allow you to process an array and it gives back an array of results. Below is an example:
var async = require('async');
var fs = require('fs');
var request = require('request');
function processUrl(url, callback){
request.head(url, function(err, res, body){
var newImageName = randomInt(100000, 999999);
var filename = 'catalog/import/'+newImageName+'.jpg';
request(url, {encoding: 'binary'}, function(error, response, body) {
fs.writeFile('image/'+filename, body, 'binary', function (err) {
if(err) return callback(err);
callback(null,filename);
});
});
});
}
function someFunctionAsDownloadedRemoteIMages(arrayOfRemoteUrls, callback){
async.map(arrayOfRemoteUrls, processUrl, callback);
}
var remoteImagesArray = ["http://example.com/1.jpg", "http://example.com/1444.jpg","http://example.com/ddsgggg.jpg"];
someFunctionAsDownloadedRemoteIMages(remoteImagesArray, function(err, localImagesArray){
if(err) //handle it
someFunctionProccess(localImagesArray);
});

Related

can't use returned value in node.js

This function returns duration:
getEventLog.getId(uuid, function(err, id){
if(err) return console.log(err)
getEventLog.getDuration(id, function(err, duration, callback){
if(err) return console.log(err)
})
});
If I add console.log(duration) below if(err) return callback(duration) I can see its value.
I'd like to use this as a variable in other functions. I tried just setting the entire function as a value like this:
var duration = getEventLog.getId(uuid, function(err, id){
if(err) return console.log(err)
getEventLog.getDuration(id, function(err, duration, callback){
if(err) console.log(err)
control.log(duration)
})
});
It didn't work though. I then thought a callback might be way but can't get one working. This was my attempt:
getEventLog.getId(uuid, function(err, id, callback){
if(err) return console.log(err)
getEventLog.getDuration(id, function(err, duration, callback){
if(err) console.log(err)
callback(duration)
})
});
How do I change this so I can use the duration value?
Additional info
If I try the call back I have
getEventLog.getId(uuid, function(err, id, callback){
if(err) return console.log(err)
getEventLog.getDuration(id, function(err, duration){
if(err)
var dur = duration
callback(dur)
})
});
var myVal = getEventLog.getId(function(dur) {
console.log("myVal: " + dur)
});
I get:
modules/getEventLog.js:36
callback(null, arrFound);
^
TypeError: callback is not a function
If I try the return I get:
getEventLog.getId(uuid, function(err, id, callback){
if(err) return console.log(err)
getEventLog.getDuration(id, function(err, duration){
if(err) return duration
})
});
var myVal = getEventLog.getId()
console.log("myVal: " + myVal)
I also get:
modules/getEventLog.js:36
callback(null, arrFound);
^
TypeError: callback is not a function
I think it might get thrown when return console.log(err) is removed.
Apologies for the silly questions but I'm new to Node.js. Where am I going wrong?
I included getEventLog below for reference:
var http = require("http");
var fs = require('fs');
var async = require('async');
var readline = require('readline')
//var db = require('./dbPool');
//get file name
var options = {
"method" : "GET",
"hostname" : "127.0.0.1",
"port" : "18080",
"path" : "/api/v1/applications/"
};
exports.getId = function (uuid, callback) {
var req = http.request(options, function (res) {
var chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
var body = JSON.parse(Buffer.concat(chunks));
var arrFound = Object.keys(body).filter(function(key) {
if (body[key].name.indexOf(uuid) > -1) {
return body[key].name;
}
}).reduce(function(obj, key){
obj = body[key].id;
return obj;
}, {});
callback(null, arrFound);
});
});
req.end();
}
exports.getDuration = function (myId, callback) {
//getEventLog.getId(function(err, id){
//get file name
var options = {
"method" : "GET",
"hostname" : "127.0.0.1",
"port" : "18080",
"path" : "/api/v1/applications/" + myId
};
var req = http.request(options, function (res) {
console.log(options)
var chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
var body = JSON.parse(Buffer.concat(chunks));
var attempts = body.attempts
var arrFound = Object.keys(body).filter(function(key) {
return attempts[0].duration;
}).reduce(function(obj, key){
obj = body.attempts[0].duration;
return obj;
}, {});
//console.log(arrFound);
callback(null, arrFound);
});
});
req.end();
// })
};
You should rethink what you want to achieve. You describe, that you want a return value, but you do not return anything. You execute a callback function (which is fine too).
A function can return a value like this:
function foo() {
return 'bar';
}
var value = foo(); // value === 'bar'
Or you can execute a callback function (mostly async) and put your "return value" as a parameter into it, so:
function foo(callback) {
// do some stuff here
var duration = 123;
callback(duration);
)
// executing foo, putting a callback function into it
foo(function(duration)) {
// duration will be 123 here
});
EDIT: Please note, that you should always only show the minimum of your code, which is needed to understand your problem.
You actually thinking and coding correctly, but your function call is wrong (which I wasn't aware until your edit).
getEventLog.getId(uuid, function(err, id, callback){
if(err) return console.log(err)
getEventLog.getDuration(id, function(err, duration){
if(err)
var dur = duration
callback(dur)
})
});
/*
* take a look at this line:
* your function requires your first parameter to be an `uuid`,
* but you simply skipped it. Now your second parameter is null,
* resulting in "callback is not a function"-error.
*/
var myVal = getEventLog.getId(function(dur) {
console.log("myVal: " + dur)
});
Just call your function correctly, by passing an uuid as first parameter and your callback function as the second.

NodeJS Async - Continue execution for multiple http requests even if some fail

I am trying to make multiple HTTP requests and cumulate, display the results in NodeJS using the following code:
const async = require('async');
const request = require('request');
function httpGet(url, callback) {
const options = {
url : url,
json : true
};
request(options,
function(err, res, body) {
console.log("invoked")
callback(err, body);
}
).on('error', function(err) {
console.log(err)
});
}
const urls= [
"http://1.2.3.4:30500/status/health/summary",
"http://5.6.7.8:30505/status/health/summary"
];
async.map(urls, httpGet, function (err, res){
if (err)
console.log(err);
else
console.log(res);
});
The problem here is, if the first request(http://1.2.3.4:30500/status/health/summary) fails (like connection refused etc.), the second one does not go through. I know that I am making a silly mistake but cannot find it. Any help appreciated !
In async.map if one of the calls passes an error to its callback, the main callback (for the map function) is immediately called with the error(this is the problem in your case). In order not to terminate on the first error, don't call the callback with err param in your httpGet.
Use async each, it receives a list of arguments and a function, and calls the function with each element, make sure in your httpGet inside on error you call the callback, without the err, this will make rest of the calls to continue even if there was an error in some of the calls. This can work for map too but, I think the more suitable function for your case is async.each, instead of map, also you can limit the number of concurrent calls with eachLimit method.
Check https://caolan.github.io/async/docs.html#each
const async = require('async');
const request = require('request');
function httpGet(url, callback) {
const options = {
url : url,
json : true
};
request(options,
function(err, res, body) {
if (err){
console.log(err);
callback();
return;
}
console.log("invoked")
callback(null, body);
}
).on('error', function(err) {
console.log(err);
callback();
});
}
const urls= [
"http://1.2.3.4:30500/status/health/summary",
"http://5.6.7.8:30505/status/health/summary"
];
async.each(urls, httpGet, function (err, res) {
}, function (err, res) {
});
If you want async.map NOT to fail fast you could do it like this
const async = require('async');
const request = require('request');
function httpGet(url, callback) {
const options = {
url : url,
json : true
};
request(options,
function alwaysReportSuccess(err, res, body) {
callback(null, {
success: !err,
result: err ? err : body
});
}
).on('error', function(err) {
console.log(err)
});
}
const urls= [
"http://1.2.3.4:30500/status/health/summary",
"http://5.6.7.8:30505/status/health/summary"
];
async.map(urls, httpGet, function alwaysOk(_, res){
console.log(res); // will be an array with success flags and results
});

a nodejs request within a loop

I am building a nodejs application.
Location.find({} ,function (err, result){
var locations = [];
result.forEach(function(listItem, i){
url = "https://maps.googleapis.com/maps/api/geocode/json?address=" + encodeURIComponent(listItem.address) + "&key=AIzaSyDQsX4Kci3xHp1xy6ZjT-5lsLNI-J-CH-8";
request(url, function(error, response, body) {
all = JSON.parse(body);
locations.push({des: result[i].placeinfo,lat: all.results[0].geometry.location.lat, lng: all.results[0].geometry.location.lng });
if (i == result.length - 1) {
res.render("index.ejs", { layout: false,locationmap:locations});
}
});
});
});
I have two problems here.
My loop runs 4 times, when i try to console.log() the i var its shows 4 time in the console.
why cant i use the request body outside of the loop i did some workaround and did the res.render inside the if statement.
You may like to use asyncjs eachSeries function but you need to install(npm install asyncjs --save) and then use like
var async = require('asyncjs');
Location.find({} ,function (err, result){
var locations = [];
async.eachSeries(result, function iteratee(listItem, callback) {
url = "https://maps.googleapis.com/maps/api/geocode/json?address=" + encodeURIComponent(listItem.address) + "&key=AIzaSyDQsX4Kci3xHp1xy6ZjT-5lsLNI-J-CH-8";
request(url, function(error, response, body) {
all = JSON.parse(body);
locations.push({des: result[i].placeinfo,lat: all.results[0].geometry.location.lat, lng: all.results[0].geometry.location.lng });
callback(error, body);
});
}, function done() {
res.render("index.ejs", { layout: false,locationmap:locations});
});
});

ExpressJS - Using Q

For a certain route, I have the following code:
router.get('/:id', function(req, res) {
var db = req.db;
var matches = db.get('matches');
var id = req.params.id;
matches.find({id: id}, function(err, obj){
if(!err) {
if(obj.length === 0) {
var games = Q.fcall(GetGames()).then(function(g) {
console.log("async back");
res.send(g);
}
, function(error) {
res.send(error);
});
}
...
});
The function GetGames is defined as follows:
function GetGames() {
var url= "my-url";
request(url, function(error, response, body) {
if(!error) {
console.log("Returned with code "+ response.statusCode);
return new Q(body);
}
});
}
I'm using the request module to send a HTTP GET request to my URL with appropriate parameter, etc.
When I load /:id, I see "Returned with code 200" logged, but "async back" is not logged. I'm also not sure that the response is being sent.
Once GetGames returns something, I want to be able to use that returned object in the route for /:id. Where am I going wrong?
Since GetGames is an async function write it in node.js callback pattern:
function GetGames(callback) {
var url= "my-url";
request(url, function(error, response, body) {
if(!error) {
console.log("Returned with code "+ response.statusCode);
return callback(null,body)
}
return callback(error,body)
});
}
Then use Q.nfcall to call the above function and get back a promise:
Q.nfcall(GetGames).then(function(g) {
})
.catch()

Empty body in a response when using request

I have the following code:
var request = require('request');
var cheerio = require('cheerio');
var URL = require('url')
var fs = require('fs')
fs.readFile("urls.txt", 'utf8', function(err, data) {
if (err) throw err;
var urls = data.split('\n');
urls = urls.filter(function(n){return n});
for(var i in urls) {
request(urls[i], function(err, resp, body) {
if (err)
throw err;
$ = cheerio.load(body,{lowerCaseTags: true, xmlMode: true});
$('item').each(function(){
console.log("----------");
console.log($(this).find('title').text());
console.log($(this).find('link').text());
console.log($(this).find('pubDate').text());
});
}).end();
}
});
and from the urls.txt file I only have the following url:
http://www.visir.is/section/?Template=rss&mime=xml
When I use wget on that url I get a response which looks like an rss feed but when I do it in the code above the body is empty. Can someone explain to me why and how can I fix this?
Update: Simply removing .end() from your original script works. end() terminates the script on callback. IMO, in 2016, I'd definitely choose Request over Needle.
Request is an odd bird, and why it's not working in your case it's giving no information in the response at all.
Try with Needle instead:
var needle = require('needle');
var cheerio = require('cheerio');
var URL = require('url')
var fs = require('fs')
fs.readFile("urls.txt", 'utf8', function(err, data) {
if (err) throw err;
var urls = data.split('\n');
urls = urls.filter(function(n){return n});
for(var i in urls) {
needle.get(urls[i], function(err, resp, body) {
if (err)
throw err;
$ = cheerio.load(body,{lowerCaseTags: true, xmlMode: true});
$('item').each(function(){
console.log("----------");
console.log($(this).find('title').text());
console.log($(this).find('link').text());
console.log($(this).find('pubDate').text());
});
});
}
});

Resources