As you know, async.parallel, defined with a such code:
async.parallel([
function (callback) {
callback(err, objects);
},
function (callback) {
callback(err, status);
},
function (callback) {
callback(err, status);
},
], function (err, results) {
//smth with results[N] array...
});
performs all the tasks all together parallel. However, I need the callback result of first function (objects, to be exact) to be avialable in 2nd and 3rd functions. In other words, first step – 1st function, second – ( 2rd + 3rd parallel with results of the 1st one). async.waterfall seems to be a bad idea 'cause:
In waterfall function can't work parallel
I can't get access to every result of stack, only to the last.
Any ideas? Thanks!
You need both waterfall and parallel.
function thing1(callback) {...callback(null, thing1Result);}
function thing2A(thing1Result, callback) {...}
function thing2B(thing1Result, callback) {...}
function thing2(thing1Result, callback) {
async.parallel([
async.apply(thing2A, thing1Result),
async.apply(thing2B, thing1Result)
], callback);
}
async.waterfall([thing1, thing2], function (error) {
//all done
});
There's no need to use async. With async you are basically black-boxing your app. Because I don't like the magic for easy tasks, vanilla js:
var f1 = function (cb){
...
cb (null, "result from f1"); //null error
};
var f2 = function (resultFromF1, cb){
...
cb (null); //null error
};
var f3 = function (resultFromF1, cb){
...
cb (null); //null error
};
var main = function (cb){
f1 (function (error, resultFromF1){
if (error) return cb ([error]);
var errors = [];
var remaining = 2;
var finish = function (error){
if (error) errors.push (error);
if (!--remaining){
//f2 and f3 have finished
cb (errors.length ? errors : null);
}
};
f2 (resultFromF1, finish);
f3 (resultFromF1, finish);
});
};
main (function (errors){
if (errors) return handleError (errors); //errors is an array
...
});
Related
After reading about async, I assumed the code below would output to the console the total of all values returned from the http/API call; but it seems to fire immediately after the first http call returns, and only shows a 'total' value equal to the first value returned from the API.
Where is my misunderstanding about how async.map works?
var http = require('https');
const
async = require('async');
var MongoClient = require('mongodb').MongoClient;
var dbUrl = "mongodb://localhost:27017/";
var total = 0;
var tokens = [ {
name : "tron"
}, {
name : 'cardano'
}, {
name : 'nucleus-vision'
}, {
name : 'ripple'
}, {
name : 'litecoin'
}, {
name : 'havven'
}];
function run() {
doStuff();
setInterval(doStuff, 1 * 60 * 1000);
};
function doStuff() {
total = 0;
async.map(tokens, httpGet, function (value){
console.log('async done ', total);
});
}
function httpGet(token, callback) {
var url = 'https://api.coinmarketcap.com/v1/ticker/' + token.name;
http.get( url,
function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
var jsonObj = JSON.parse(body);
var price = parseFloat(jsonObj[0].price);
total += price;
MongoClient.connect(dbUrl, function(err, db) {
if (err)
throw err;
var dbo = db.db("crypto");
dbo.collection("tick").insertOne(jsonObj[0],
function(err, res) {
if (err)
throw err;
db.close();
});
});
callback(price);
});
}).on('error', function(e) {
console.log("Got an error: ", e);
});
};
run();
callback that is passed to an iteratee (httpGet) is used incorrectly. The first argument (price) is considered an error. From the docs:
If iteratee passes an error to its callback, the main callback (for the map function) is immediately called with the error.
So
callback(price);
should rather be
callback(null, price);
So async does not halt after the first iteration.
I believe there are two separate problems here:
As you may know, we cannot use return statements in asynchronous code like we would in synchronous code, which is why we use callbacks instead. Node-style callbacks are on the form function (err, result) {}, where the first parameter is the error (if any) and the second the result of the function (the return value). According to the docs,
Async.map(coll, iteratee, callback) will stop the execution if the
iteratee passes an error to its callback.
As your iteratee-function is calling its callback as such: callback(price), you're effectively stopping execution, as price is passed as the error parameter. What you want to do to "return" the price variable, is to call the callback as so: callback(null, price)
Typically, map-functions are used for
appl[ying] a given function to each element of a list, returning a list of results in the same order.
The map function of the async library does the same, IE: iterates through an array and returns an array of the resulting items, just like the normal map (below) method does.
[1, 2, 3].map(function (nbr) { return nbr*2 }) // returns [2, 4, 6]
The result parameter of your callback (IE, the third parameter to async.map) will be called with an array of prices, and not the summed value of the prices.
async.map(tokens, httpGet, function (error, total) {
console.log(error); // prints undefined (unless there was an error)
console.log(total); // prints an array of prices
});
For summing the values, I would recommend the reduce function, or simply sum the values returned as a result.
I have functions in different place.(this is common function it will use many place.
So I tried to call the function using async parallel but its not working.
I attached the my code following:L
var userId = 33;
async.parallel({
video(userId, callback) {
callback(null, callback);
},
video1(userId, callback) {
callback(null, callback);
}
}, function(err, results) {
console.log(results);
});
function video (userId, callback) {
client.query("select youtube_id,title,id,thumbnail,search_item from resource order by random() limit 1 ", function(err, video) {
callback(err, video);
});
}
function video1(userId, callback) {
client.query("select youtube_id,title,id,thumbnail,search_item from resource1 order by random() limit 1 ", function(err, video1) {
callback(err, video1);
});
}
The {fnName()} syntax is a function declaration. You want to use a function call ... in fact you can just use async.parallel({video, video1}, (err, results) => console.log(results) to achieve the same effect... except that you also need to pass in userId. So what you actually intend is to call the function rather than declare it:
async.parallel([
callback => video(userId, callback),
]);
You need to pass function references to async.parallel() and you need to pass an array, not an object. You want the code to have this form:
async.parallel([fn1, fn2], finalCallback);
See the async.parallel() doc here for specific examples of the syntax you want to use.
Where fn1, fn2 and finalCallback are all function references that can be called by the async.parallel() infrastructure, not something you call yourself.
Change your code to this:
async.parallel([
function(callback) {
client.query("select youtube_id,title,id,thumbnail,search_item from resource order by random() limit 1 ", function(err, video) {
callback(err, video);
}),
},
function(callback) {
client.query("select youtube_id,title,id,thumbnail,search_item from resource order by random() limit 1 ", function(err, video1) {
callback(err, video1);
});
}
], function(err, results) {
console.log(results);
});
Presumably, your two database queries should be different somehow (your code showed them identical).
If your functions are already defined somewhere, then you can pass an anonymous function reference that will call them. You need to do this if your functions don't have the exact same calling convention that async.parallel() requires. You can do that like this:
async.parallel([function(callback) {
video(userId, callback);
}, function(callback) {
video1(userId, callback);
}], function(err, results) {
console.log(results);
});
I'm trying to understand how to wait for http requests to finish in node. I want to make two http requests and use the results in a function that gets called after the http requests are finished.
I'm using async and request and have been using async.series as following:
var request = require("request");
var express = require('express');
var async = require("async");
app.get('/rx', function(req, res) {
var drug1 = req.query.drug1;
var drug2 = req.query.drug2;
console.log("in rx")
console.log(drug1);
console.log(drug2);
var id1 = '';
var id2 = '';
/*part of code I'm concerned with*/
async.series([
function(callback) {
id1 = getID(drug1);
console.log("function one");
console.log(id1);
callback();
},
function(callback) {
id2 = getID(drug2);
console.log("function two");
console.log(id2);
callback();
}
],
function(err, results) {
console.log(id1);
console.log(id2);
request("http://rxnav.nlm.nih.gov/REST/interaction/interaction.json?list?rxcuis=" + id1 + "&sources=" + id2, function(error, response, body) {
console.log("finished!");
res.json(body);
});
});
});
//returns an int ID
function getID(drugName) {
request("http://rxnav.nlm.nih.gov/REST/Prescribe/rxcui.json?name=" + drugName, function(error, response, body) {
var content = JSON.parse(body);
var id = parseInt(content.idGroup.rxnormId);
console.log("in getID function");
console.log(id);
return id;
});
}
The console output shows:
in rx
advil
ibuprofen
seriesone
undefined
two
undefined
undefined
undefined
finished!
GET /rx?drug1=advil&drug2=ibuprofen 304 345ms
in getID function
153010
in getID function
5640
I want to wait until each http request function is completed, and then proceed to the next portion of code. How do I achieve this?
This question (or variants thereof) has been asked more than 1000 times here on StackOverflow. Therefore I'm not going to explain it but you can search "return async" on this site (the search input at the top right corner) if you want to know more.
The basic problem is that it's impossible to return values from an async function (ever wonder why they accept callbacks?).
For your specific case, you need to change getId() to:
//returns an int ID
function getID(drugName, callback) {
request("http://rxnav.nlm.nih.gov/REST/Prescribe/rxcui.json?name=" + drugName, function(error, response, body) {
var content = JSON.parse(body);
var id = parseInt(content.idGroup.rxnormId);
console.log("in getID function");
console.log(id);
callback(null,id); // <--- this is how you return async values
});
}
Note: the null is because functions in the async.js family expects the first argument to the callback to be an error. So if there are no errors pass null. This by the way is a standard node.js practice.
Then inside async.series() you do:
async.series([
function(callback) {
getID(drug1,callback);
},
function(callback) {
getID(drug2,callback);
}
],
function(err, results) {
console.log(results[0]); // id1
console.log(results[1]); // id2
// do the rest..
});
I have a main in nodejs for my program where I need to use my result calculated in a module, but my I don't have the right result.
var myJSONClient = {
"nombre" : "<nombre_cliente>",
"intervalo" : [0,0]
};
var intervalo = gestionar.gestion(myJSONClient,vector_intervalo);
console.log("intervalo: "+intervalo); //return undefined
And this is the module
var gestion = function(myJSON,vector_intervalo) {
var dburl = 'localhost/mongoapp';
var collection = ['clientes'];
var db = require('mongojs').connect(dburl, collection );
var intervalo_final;
function cliente(nombre, intervalo){
this.nombre = nombre;
this.intervalo = intervalo;
}
var cliente1 = new cliente(myJSON.nombre,myJSON.intervalo);
db.clientes.save(cliente1, function(err, saveCliente){
if (err || !saveCliente) console.log("Client "+cliente1.nombre+" not saved Error: "+err);
else {
console.log("Client "+saveCliente.nombre+" saved");
intervalo_final = calculate(vector_intervalo);
console.log(intervalo_final); //here I can see the right content of the variable intervalo_final
}
});
setTimeout(function(){
console.log("pause");
},3000);
console.log(intervalo_final); //result not correct
return intervalo_final;
}
exports.gestion = gestion;
I know that node execute my return without wait the end of my function, for this I can't see the right result, but how can I force my program to wait the end of my function?
I tried with the setTimeout function but wasn't the right way.
You must implement your function just like the other async functions from the API!
First step : give callback to function
var gestion = function(myJSON,vector_intervalo, callback) {
Second step : when the async process is over call callback passing the result (you don't need the return line)
console.log(intervalo_final); //here I can see...
callback(intervalo_final);
Step three: use your function in an async way
gestionar.gestion(myJSONClient,vector_intervalo, function(result){
console.log(result);
});
In async JS you can't return a value the way it seems you trying to do. You need to pass a callback function from your main program when calling gestionar.gestion() (you can add it as a third argument).
Your code sample won't work because function gestion() returns immediately, before intervalo_final content is set.
Something like this:
gestionar.gestion(myJSONClient,vector_intervalo, function callback(intervalo) {
// This is the callback function
console.log("intervalo: " + intervalo);
});
And then within the function:
var gestion = function(myJSON,vector_intervalo, callback) {
...
db.clientes.save(cliente1, function(err, saveCliente) {
if (err || !saveCliente) {
console.log("Client "+cliente1.nombre+" not saved Error: "+err);
if (callback) callback(); // execute callback function without arguments
}
else {
console.log("Client "+saveCliente.nombre+" saved");
intervalo_final = calculate(vector_intervalo);
console.log(intervalo_final);
if (callback) callback(intervalo_final); // your callback function will be executed with intervalo_final as argument
}
});
Also, I highly recommend reading some async javascript tutorial, like http://javascriptissexy.com/understand-javascript-callback-functions-and-use-them/
And Felix's Node.js Guide: http://nodeguide.com/
I'm writing a simple request handler to return a pair of css files. Using fs.readFileSync this was easy. However, I'm having difficulty accomplishing the same task using the async version of readFile. Below is my code. Having my response.write() method calls split among two different callbacks seems to be problematic. Can someone point out what I've done wrong? Interestingly this code works if I put response.end() inside of the first else statement. However, that creates a problem in that the second css file does not get returned (because response.end() has already been fired).
function css(response) {
response.writeHead(200, {"Content-Type": "text/css"});
fs.readFile('css/bootstrap.css', function(error, content){
if(error){
console.log(error);
}
else{
response.write(content);
}
});
fs.readFile('css/bootstrap-responsive.css', function(error, content){
if(error){
console.log(error);
}
else{
response.write(content)
}
});
response.end();
}
The primary issue with what you have is that response.end() gets called right away. You need to only call it after the files have done their response.write calls.
The easiest way would be to use a control flow library. Managing multiple asynchronous callbacks is generally complicated.
https://github.com/joyent/node/wiki/modules#wiki-async-flow
I'm going to use the async library because it's the one I know best.
var fs = require('fs');
var async = require('async');
function css(response) {
response.writeHead(200, {"Content-Type": "text/css"});
async.eachSeries(
// Pass items to iterate over
['css/bootstrap.css', 'css/bootstrap-responsive.css'],
// Pass iterator function that is called for each item
function(filename, cb) {
fs.readFile(filename, function(err, content) {
if (!err) {
response.write(content);
}
// Calling cb makes it go to the next item.
cb(err);
});
},
// Final callback after each item has been iterated over.
function(err) {
response.end()
}
);
}
If you want to accomplish this without a library, or just want another way, this is how I would do it more directly. Basically you keep a count and call end once both file reads have finished.
function css(response) {
response.writeHead(200, {"Content-Type": "text/css"});
var count = 0;
var handler = function(error, content){
count++;
if (error){
console.log(error);
}
else{
response.write(content);
}
if (count == 2) {
response.end();
}
}
fs.readFile('css/bootstrap.css', handler);
fs.readFile('css/bootstrap-responsive.css', handler);
}
You can simply rely on html5 Promise. The code can be as simple as follows:
var promises= ['file1.css', 'file2.css'].map(function(_path){
return new Promise(function(_path, resolve, reject){
fs.readFile(_path, 'utf8', function(err, data){
if(err){
console.log(err);
resolve(""); //following the same code flow
}else{
resolve(data);
}
});
}.bind(this, _path));
});
Promise.all(promises).then(function(results){
//Put your callback logic here
response.writeHead(200, {"Content-Type": "text/css"});
results.forEach(function(content){response.write(content)});
response.end();
});
There's a simple common solution to get them all with an one callback.
You can place it anywhere in your project to reuse in many different cases.
var FS = require('fs');
/**
* Abstract helper to asyncly read a bulk of files
* Note that `cb` will receive an array of errors for each file as an array of files data
* Keys in resulting arrays will be the same as in `paths`
*
* #param {Array} paths - file paths array
* #param {Function} cb
* #param {Array} errors - a list of file reading error
* #param {Array} data - a list of file content data
*/
function FS_readFiles (paths, cb) {
var result = [], errors = [], l = paths.length;
paths.forEach(function (path, k) {
FS.readFile(path, function (err, data) {
// decrease waiting files
--l;
// just skip non-npm packages and decrease valid files count
err && (errors[k] = err);
!err && (result[k] = data);
// invoke cb if all read
!l && cb (errors.length? errors : undef, result);
});
});
}
Just put inside it a bulk of files and it will returns to you each of them as a buffer.
Simple example:
var cssFiles = [
'css/bootstrap.css',
'css/bootstrap-responsive.css'
];
function css(response) {
FS_readFiles(cssFiles, function (errors, data) {
response.writeHead(200, {"Content-Type": "text/css"});
data.forEach(function (v) {
response.write(v);
});
response.end();
});
}
Offtopic: Btw, requests like this you better to cache on front-end proxy server like nginx or varnish. It's never change.
const fs = require('fs');
function readFilePromise(fileName) {
return new Promise(function (resolve, reject) {
fs.readFile(fileName, 'utf-8', function(err, data){
if(err){reject(err)} else {
resolve(data)
}
})
})
}
Promise.all([readFilePromise("abc.txt"), readFilePromise("dec.txt")]).then(function(out){
console.log(out)
})
Async is an awesome lib. However the standard for these things is moving in the direction of promises for handling multiple asynchronous operations. In fact in ECMAScript6 this will be a standard part of the library. There are several libraries that implement promises including JQuery. However, for node, I like to use 'q'
Here is the same code using promises: One note.. you might want to move the first writeHead call to coincide with the first successful read.
var Q = require('q');
function css(response) {
response.writeHead(200, {"Content-Type": "text/css"});
var defer = Q.defer();
fs.readFile('css/bootstrap.css', function(error, content){
if(error){
defer.reject(error)
}
else{
response.write(content);
defer.resolve();
}
});
defer.promise.then(function() { //this gets executed when the first read succeeds and is written
var secondDefer = Q.defer();
fs.readFile('css/bootstrap-responsive.css', function(error, content){
if(error){
secondDefer.reject(error);
}
else{
response.write(content);
secondDefer.resolve();
}
});
return secondDefer.promise;
},
function(error) { //this gets called when the first read fails
console.log(error);
//other error handling
}).
done(function() {
response.end();
},
function(error) { //this is the error handler for the second read fails
console.log(error);
response.end(); //gotta call end anyway
});
}