my api only allows a maximum of 10 requests every second. I am trying to keep under this rate limit with async library. I have tried multiple functions but none of them work.
Pauseconnect and connectStream opens an eventstream for each item. let me know if you need to see their code.
async.queue --doesn't wait.
var cargo = async.queue(function (task, callback) {
setTimeout(
connectStream(task)
, 50000);
callback();
}, 1);
for(var j = 0; j < TeamList.length; j++) {
cargo.push(TeamList[j], function (err) {
});
async.eachLimit --stops at 5 and doesn't progress
async.eachLimit(TeamList, 5, pauseConnect, function(err){
if(err) {console.log(err);}
});
rate-limiter -- runs through all of them without waiting
limiter.removeTokens(1, function() {
for(var i=0; i< TeamList.length; i++){
connectStream(TeamList[i]);
}
});
async.each-- doesn't wait just runs through all of them
async.each(TeamList pauseConnect, function(err){
if(err) {console.log(err);}
});
You're missing the callback in the async.each, it should be (for example)...
async.each(TeamList pauseConnect, function(err, callback) {
if(err) {
console.log(err);
}
return callback();
});
If anyone is curious this worked for me
async.eachLimit(TeamList, 5, function(team, callback){
setTimeout(function(){
connectStream(team);
callback();
}, (5000));
}, function(err){
if(err) {console.log(err);}
});
Related
Here's a simple example of the use of async.parallel:
var fakeTimer = 0;
async.parallel({
one: function(callback) {
if (fakeTimer < 2) {
callback(new Error('too soon!'), null);
fakeTimer++;
} else {
callback(null, 'I am one');
}
},
two: function(callback) {
callback(null, 'I am two');
}
}, function(err, results) {
if (err) {
console.log('failed!');
} else {
console.log(results);
}
});
When this runs, of course it always ends in failure. What I'd like to do is keep retrying until fakeTimer has become large enough that the one function succeeds.
So either the whole async.parallel could be retried e.g. 5 times, or just the one function. I know that there is the async.retry feature, but I just can't get my head around how to combine that with async.parallel to achieve what I want.
I think ideally the whole async.parallel should be retried, so that it works if the error happens in any of the parallel branches, but it would be great to see an example of an overall retry and a per-branch retry.
The following seems to work:
var fakeTimer = 0;
var parallelFunctions = {
one: function(callback) {
if (fakeTimer < 2) {
callback(new Error('too soon!'), null);
fakeTimer++;
} else {
callback(null, 'I am one');
}
},
two: function(callback) {
callback(null, 'I am two');
}
};
var doThemInParallel = function(callback) {
async.parallel(parallelFunctions, function(err, results) {
callback(err, results);
});
};
var retries = 2; // must be > 2 to succeed
async.retry(retries, doThemInParallel, function(err, results) {
console.log(err, results);
});
I've got following code now:
exports.listByUser = function(req, res) {
Attack.find({user: req.user._id}, function(err, attacks) {
if(err)
return next(err);
for(var i in attacks) {
attacks[i].evaluateFight();
}
res.json(attacks);
});
};
the main problem is that attacks[i].evaluateFight() is called asynchronously, I want to transform it to make sure that [i-1] iteration is done ... and finally call res.json(attacks). I think, it can be done with async, but I don't know how :( Something like this should work, but how can I call attacks.method?
async.eachSeries(attacks, function (callback) {
//something??
callback();
}, function (err) {
if (err) { throw err; }
res.json(attacks);
});
You can leverage async whilst method call to implement the same. However, there is question I have about the callback of evaluateFight because if it is executed asynchronously then there has to be some callback associated with it which will notify if the previous call is succeeded.
The example code can be as follows assuming evaluateFight returns a callback when completed -
exports.listByUser = function(req, res) {
Attack.find({user: req.user._id}, function(err, attacks) {
if(err)
return next(err);
var attacksLength = attacks.length;
var count = 0;
async.whilst(function () {
return count < attacksLength;
},
function (callback) {
attacks[count].evaluateFight(function(err, result){
count++;
callback();
}); // assuming it returns a callback on success
},
function (err) {
// all the iterations have been successfully called
// return the response
res.json(attacks);
});
};
I have a requirement to make several API requests and then do some processing on the combines result sets. In the example below, you can see that 3 requests are made (to /create) by duplicating the same request code however I would like to be able to specify how many to make. For example, I may wish to run the same API call 50 times.
How can I make n calls without duplicating the API call function n times?
async.parallel([
function(callback){
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
},
function(callback){
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
},
function(callback){
request.post('http://localhost:3000/api/store/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
}
],
function(err, results){
if (err) {
console.log(err);
}
// do stuff with results
});
First, wrap the code that you want to call many times in a function:
var doRequest = function (callback) {
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err);
}
callback(null, res.body.id);
});
}
Then, use the async.times function:
async.times(50, function (n, next) {
doRequest(function (err, result) {
next(err, result);
});
}, function (error, results) {
// do something with your results
}
Create an array with as many references to the function as you need tasks in your workload. Then pass them to async.parallel. For example:
var async = require("async");
var slowone = function (callback) {
setTimeout(function () {
callback(null, 1);
}, 1000);
};
async.parallel(
dd(slowone, 100),
function (err, r) {
console.log(JSON.stringify(r));
}
);
// Returns an array with count instances of value.
function dd(value, count) {
var result = [];
for (var i=0; i<count; i++) {
result.push(value);
}
return result;
}
Note again that there is only one instance of the slow running function, in spite of there being many references to it.
I have been coding in Node.js:
var sql = ' SELECT 1 AS re';
for(var i=0;i<10;i++){
connection.query(sql,function(err,rows){
if(err){connection.rollback(function (e) {throw err;});}
else{
console.log('foo');
}
});
}
setTimeout(function(){ console.log('b }, 50);
Can I lined output up exactly? Like: foo --> bar
You will need to wrap your code either in async or in a form of a promise.
For Instance:
var sql = 'SELECT 1 AS re';
var count = 0;
async.whilst(
function() {
return count < 10;
},
function(done) {
count++;
connection.query(sql, function(err, rows) {
if(err)
connection.rollback(function(e) { done(); });
else {
console.log('foo');
done();
}
});
},
function(err) {
console.log('bar');
});
Now this will be sequential. Again, if this isn't what you were asking. I'd suggest to please update your question, so that we have a better idea of what the problem is, or at least what are your intentions of doing.
I wrote a function to copy a directory to another... But there's a problem : I use callback function to send the copied size. This callback comes too early (before the end of the copy). I think the problem is that the process is asynchronous. Can you help me?
var fs=require('fs');
var copyDir=function copyDir(from, to, callback){
if(!fs.existsSync(to)){
fs.mkdirSync(to);
}
console.log(from+" ==> "+to);
var count = 0;
fs.readdir(from, function(err,files){
for(var i=0;i<files.length;i++){
var f = from+"/"+files[i];
var d = f.replace(from, to);
console.log(f+" ("+i+")"+ " : "+d);
if(!fs.existsSync(d)){
if(!fs.statSync(f).isFile()){
//fs.mkdirSync(f.replace(from, to));
count += fs.statSync(f).size;
console.log(f + " will make an inception!")
copyDir(f, f.replace(from, to), function(err, cp){callback(err, cp)});
}else{
var size = fs.statSync(f).size;
copyFile(f, f.replace(from, to), function(err){
if(err) callback(err, count)
});
count += size;
callback(null, count);
}
}
}
});
}
function copyFile(source, target, cb) {
fs.readFile(source, function (err, data) {
if (err) throw err;
fs.writeFileSync(target, data, function (err, data){
if(err) throw err;
cb(null, fs.statSync(source).size); //This callback comes before the copy end.
});
});
}
exports.copyDir = copyDir;
copyDir is called by:
io.sockets.on('connection', function(socket){
console.log('connection');
socket.on('startCopy', function(data){
sizeDir('templates', function(e, r){
copyDir('templates', 'tmp', function(err, cp){
console.log("copy % " + Math.round(100*cp/r));
socket.emit('copy', {prog: Math.round(100*cp/r)});
});
});
});
});
You can rewrite your else code with following:
(function() {
var size = fs.statSync(f).size;
copyFile(f, f.replace(from, to), function(err){
if(err) {
callback(err, count);
return;
}
count += size;
callback(null, count);
});
})();
But, you have a lot of synchronous function in your code. You should know about all caveats of this approach. This article may be helpful