node async module: combine parallel with retry - node.js

Here's a simple example of the use of async.parallel:
var fakeTimer = 0;
async.parallel({
one: function(callback) {
if (fakeTimer < 2) {
callback(new Error('too soon!'), null);
fakeTimer++;
} else {
callback(null, 'I am one');
}
},
two: function(callback) {
callback(null, 'I am two');
}
}, function(err, results) {
if (err) {
console.log('failed!');
} else {
console.log(results);
}
});
When this runs, of course it always ends in failure. What I'd like to do is keep retrying until fakeTimer has become large enough that the one function succeeds.
So either the whole async.parallel could be retried e.g. 5 times, or just the one function. I know that there is the async.retry feature, but I just can't get my head around how to combine that with async.parallel to achieve what I want.
I think ideally the whole async.parallel should be retried, so that it works if the error happens in any of the parallel branches, but it would be great to see an example of an overall retry and a per-branch retry.

The following seems to work:
var fakeTimer = 0;
var parallelFunctions = {
one: function(callback) {
if (fakeTimer < 2) {
callback(new Error('too soon!'), null);
fakeTimer++;
} else {
callback(null, 'I am one');
}
},
two: function(callback) {
callback(null, 'I am two');
}
};
var doThemInParallel = function(callback) {
async.parallel(parallelFunctions, function(err, results) {
callback(err, results);
});
};
var retries = 2; // must be > 2 to succeed
async.retry(retries, doThemInParallel, function(err, results) {
console.log(err, results);
});

Related

Asynchronous function - node.js

I have a question about asynchronous function. Here my function "My_function":
function My_function (my_name, callback){
stmt = db.prepare ("SELECT number_table1 from my_table1 WHERE user=?");
stmt.bind(my_name);
stmt.get(function(error,row){
if(error){
throw err;
}
else{
if(row){
callback(number_table1);
}
else{
console.log("error");
}
}
});
}
Work fine but I have 2 tables and I need do other query and I need add two numbers so... in my function I need do too this query:
stmt = db.prepare ("SELECT number_table2 from my_table2 WHERE user=?");
and finally return back in my callback "number_table1 + number_table2".
Somebody know how to solve it? Thanks in advance.
Best regards!
In cases like your's I like to use the async module, because the code will be more legible. Example:
var async = require('async');
function My_function(my_name, callback) {
var stmt = db.prepare("SELECT number_table1 from my_table1 WHERE user=?");
stmt.bind(my_name);
stmt.get(function (error, row) {
if (error) {
callback(error, null);
} else {
if (row) {
callback(null, number_table1);
} else {
callback(new Error("row not found"), null);
}
}
});
}
//if you need the results in a specific order, you can use "series" instead of "parallel".
async.parallel(
//array of functions
[
function (callback) {
My_function('firstName', callback);
},
function (callback) {
My_function('secondName', callback);
}
],
//results when all functions ends
//the results array will equal [number_table1, number_table2], but the order can be different, because of the parallelism
function (err, results) {
if (err) {
//handle the err
} else {
//do something
}
}
);
Docs:
http://caolan.github.io/async/docs.html#parallel or
http://caolan.github.io/async/docs.html#series
You need to synchronize the functions so that you can be sure both their results are ready before calling back. You can do this using promises: https://www.promisejs.org/
Make two regular functions (no callbacks), one for each query (function1, function2)
Make both return a promise
Then you can do
function My_function(my_name) {
var value1;
function1(my_name)
.then(function(resultFromFunction1) {
value1 = resultFromFunction1;
return function2(my_name);
})
.then(function(resultFromFunction2) {
var result = value1 + resultFromFunction2;
return result;
});
}
}
Make sure to catch errors and handle different outcomes, what I presented is its simplest form.
Update
Here is an example of a function doing a query and returning a promise
function1 = function(user) {
return new Promise(function (resolve, reject) {
pool.getConnection(function (err, connection) {
if(err) {
reject ({status : false, message : "Error in connection database"});
} else {
connection.query('SELECT number_table1 from my_table1 WHERE user=?', [user], function(err, rows){
connection.release();
if(!err) {
resolve ({status: true, message: rows});
} else {
reject ({status: false, message: err});
}
});
}
});
});
}
Make the table names function parameters. Convert that function to use async/await or promise. Use Promise.all to run both queries.

Node.js callbacks and recursion

I don't understand how to call a function recursively in node.js for example:
var releaseStock = function (callback) {
getItems(function (err, items) {
if (err) {
return callback(err);
} else {
if (items) {
return callback(items);
} else {
setTimeout(function() {
releaseStock(callback);
}, 5000);
}
}
});
};
How can i make it work?
I'm not entirely sure what you want to do, but I suspect it is something along the lines of:
var releaseStock = function(callback) {
// get items from somewhere:
var items = getItems();
if (!items) {
// if there are no items, try again (recurse!):
return releaseStock(callback);
}
// if there are items, give them to the callback function:
return callback(items);
};

Call same function many times and process combined result set

I have a requirement to make several API requests and then do some processing on the combines result sets. In the example below, you can see that 3 requests are made (to /create) by duplicating the same request code however I would like to be able to specify how many to make. For example, I may wish to run the same API call 50 times.
How can I make n calls without duplicating the API call function n times?
async.parallel([
function(callback){
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
},
function(callback){
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
},
function(callback){
request.post('http://localhost:3000/api/store/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
}
],
function(err, results){
if (err) {
console.log(err);
}
// do stuff with results
});
First, wrap the code that you want to call many times in a function:
var doRequest = function (callback) {
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err);
}
callback(null, res.body.id);
});
}
Then, use the async.times function:
async.times(50, function (n, next) {
doRequest(function (err, result) {
next(err, result);
});
}, function (error, results) {
// do something with your results
}
Create an array with as many references to the function as you need tasks in your workload. Then pass them to async.parallel. For example:
var async = require("async");
var slowone = function (callback) {
setTimeout(function () {
callback(null, 1);
}, 1000);
};
async.parallel(
dd(slowone, 100),
function (err, r) {
console.log(JSON.stringify(r));
}
);
// Returns an array with count instances of value.
function dd(value, count) {
var result = [];
for (var i=0; i<count; i++) {
result.push(value);
}
return result;
}
Note again that there is only one instance of the slow running function, in spite of there being many references to it.

Node.js callback ordering

I have been coding in Node.js:
var sql = ' SELECT 1 AS re';
for(var i=0;i<10;i++){
connection.query(sql,function(err,rows){
if(err){connection.rollback(function (e) {throw err;});}
else{
console.log('foo');
}
});
}
setTimeout(function(){ console.log('b }, 50);
Can I lined output up exactly? Like: foo --> bar
You will need to wrap your code either in async or in a form of a promise.
For Instance:
var sql = 'SELECT 1 AS re';
var count = 0;
async.whilst(
function() {
return count < 10;
},
function(done) {
count++;
connection.query(sql, function(err, rows) {
if(err)
connection.rollback(function(e) { done(); });
else {
console.log('foo');
done();
}
});
},
function(err) {
console.log('bar');
});
Now this will be sequential. Again, if this isn't what you were asking. I'd suggest to please update your question, so that we have a better idea of what the problem is, or at least what are your intentions of doing.

async - callback already used error

I'm getting a 'callback already used' error and I don't know why. I am using async and want to chain two functions because the second function depends on the first function to complete.
I'm new-ish to Node.js and still wrapping my head around async/callbacks. Thanks so much for helping out.
getCdn takes in cnames, and if the cname is part of a CDN it pushes the results into a global variable called cdnAttrs.
function getCdn(cnameDict, callback) {
// cdnAttributes contains associative array with each web attribute: {name_in_db : code_snippet_to_find_in_cname}
for (var key in cdnAttributes) {
if (cdnAttributes.hasOwnProperty(key)) {
var snippet = -1;
// some technologies contain multiple code snippets, in that case they are stored as array. Single code snippets are stored as string
if (!Array.isArray(cdnAttributes[key])) {
snippet = cnameDict['cname'].indexOf(cdnAttributes[key])
}
else {
// check each code snippet within the array, if any match the source code, update 'snippet'
for (var n = 0; n < cdnAttributes[key].length; n++) {
var val = cnameDict['cname'].indexOf(cdnAttributes[key][n])
if (val > -1) {
snippet = val
}
}
}
// if attribute found in tag, create cdnAttrs[cdn] = [{from: hostname, proof: cname}, {from: hostname2, proof: cname2}, ...]
if (snippet > -1) {
try {
cdnAttrs[key].push(cnameDict);
}
catch (e) {
cdnAttrs[key] = [];
cdnAttrs[key].push(cnameDict);
}
callback();
} else {
callback();
}
} else {
callback();
}
}
}
My async function looks like this:
async.series([
// THIS FUNCTION WORKS FINE...
function(callback) {
async.each(toCheck, function(hostname, callback) {
getCname(hostname, callback);
},callback);
},
// THIS FUNCTION RETURNS RETURNS Error("Callback was already called.")
function(callback) {
async.each(toCheckCnames, function(cnameDict, callback) {
getCdn(cnameDict, callback);
},callback);
}
], function(err){
if(err) {
console.log('ERROR');
}else{
console.log('toCheckCnames is done: '+JSON.stringify(toCheckCnames));
console.log('cdnAttrs is done: '+JSON.stringify(cdnAttrs));
}
})
the getCnames function works:
function getCname(hostname, callback){
dns.resolve(hostname, 'CNAME', function(error, cname) {
if (cname) {
toCheckCnames.push({from: hostname, cname: cname[0]});
callback();
}
// if not CNAMEd, check SOA on www.domain.com and domain.com
else {
if (hostname.slice(0,4) == 'www.') {
hostname = hostname.slice(4);
}
nativedns.resolve(hostname, 'SOA', function(error, records) {
if(!error && records) {
toCheckCnames.push({from: hostname, cname: records[0]['primary']});
callback();
}
else if (!error) {
hostname = 'www.'+ hostname
nativedns.resolve(hostname, 'SOA', function(error, records) {
if (!error) {
toCheckCnames.push({from: hostname, cname: records[0]['primary']});
callback();
}
else callback()
});
}
else callback()
});
}
});
}
Your getCdn function is a loop that will call the callback after each iteration. If calling the callback is intended to stop the loop execution, you can do return callback(). Otherwise you need to reorganize your code to only call the callback once when the function is done.
UPDATE:
You can also simplify your async.each calls:
// Was this
async.each(toCheck, function(hostname, callback) {
getCname(hostname, callback);
},callback);
// Could be this
async.each(toCheck, getCname, callback);

Resources