I have folowing script
var email_list = ['email1#email.com', 'email2#email.com',....'email100#email.com'];
for(i=0;i<email_list.length;i++){
if(checkEmail(email_list[i])){
//do processing save in db and email to email addresses.
}
}
This code will be blocking in nodejs how to make this non blocking?
You can do this without blocking the event loop at all, by using a recursive loop. This way what you end up with is only launching one database worker per call, at a give time. Assuming the database work you were doing was asynchronous, your code didn't really block the event loop. But the foor loop still launched a bunch of workers simultaneously, which will tend to clog the event loop(not block it). And you are right in that it is blocking the event loop while your for loop is counting from 0, to whatever the size of your array is. The following does exactly the same thing, but you only launch one database worker at a time(good), and you never count from 0 to length. Each worker is popped off the list after the work on the current email is done, and your global event loop is left to process other things, not email_list.length database requests simultaneously.
var email_list = ['email1#email.com', 'email2#email.com', 'email100#email.com'];
function checkEmailList(emails, emailCallBack, completionCallback) {
var someDataCollectdOverAllEmails = '';
function checkEmailAsync(email) {
db.doSomeDBWorkAsync(email, function (data) {
someDataCollectdOverAllEmails += data;
if (email_list.length) {
checkEmail(email_list.pop()); //If there are still emails to be checked, check the next one ine line
} else {
completionCallback(someDataCollectdOverAllEmails);//IF not, call the completionCallBack
}
emailCallBack(data);
});
}
checkEmailAsync(emails.pop());
}
function logIndividualEmailData(data) {
console.log('Sningle Email: ' + data);
}
function logGlobalEmailData(data) {
console.log('All Email Data: ' + data);
}
checkEmailList(email_list, logIndividualEmailData, logGlobalEmailData);
Process.nextTick example
process.nextTick(function () {
'use strict';
console.log('printed second');
while (true);
});
process.nextTick(function () {
'use strict';
console.log('never printed');
});
console.log('printed first');
Note however that in the example below, despite the fact that loopForever will run forever, it still allows both of our files to be read out. If we just had while(true) it would of course block and not allow this and one of our files data would not be printed out.
var files = ['blah.js', 'file.js'];
for(var i = 0; i < files.length; i++) {
fs.readFile(files[i], function (err, data) {
console.log('File data' + data);
function loopForver(loop) {//asynchronously loop forever, pretty cool, but only useful for really specific situations!
process.nextTick(function () {
if(loop) {
console.log('looping');
loopForver(true);
}
});
}
loopForver(true);
});
}
If I need to do stuff after the emails all send, I use the async library (docs), which provides some useful functions for control flow.
You will still need to rewrite checkEmail(email) into checkEmail(email, callback) as #S.D. suggests. In checkEmail you will want to call callback after everything is completed. This probably means that you will nest callbacks, calling the second async thing (sending the email) only after the first (db query) has completed successfully.
I also suggest that you follow convention by using the first callback argument as an err parameter. If you callback(null) you are explicitly saying 'there was no error'. #S.D.'s solution suggests instead callback(ok) which is the opposite of convention.
Here is an example showing a couple nested asynchronous functions and the async library.
edit - use async.eachLimit instead of async.each so you don't execute all 100 calls simultaneously
(function main(){
var emails = ["a#b", "c#d"];
var async = require('async');
async.eachLimit(
emails // array to iterate across
,10 // max simultaneous iterations
,checkEmail // an asynchronous iterator function
,function(err){ // executed on any error or every item successful
console.log('Callback of async.eachLimit');
if(err){
console.log('Error: '+err)
} else {
console.log('All emails succeeded');
};
}
);
console.log('Code below the async.eachLimit call will continue executing after starting the asynchronous jobs');
})();
function checkEmail(email, callback){
fetchFromDb(email, function(err, obj){
if(err){ return callback(err) };
sendEmail(email, function(err, obj){
if(err){ return callback(err)};
console.log('Both fetchFromDb and sendEmail have completed successfully for '+email);
callback(null);
});
});
};
function fetchFromDb(email, callback){
process.nextTick(function(){ // placeholder, insert real async function here
callback(null);
});
};
function checkEmail(email, callback){
process.nextTick(function(){ // placeholder, insert real async function here
callback(null);
});
};
Related
today my question is actually about the problem which happens due to loop which is not working properly but anyway, i am quite open to ideas if i have issues about designing.
app.post('/admin/earning/:multiplier', function (req, res) {
var multiplier = req.params.multiplier;
var now = new Date();
var List=[];
var dailySaving;
// Getting the investors emails(key) from the collection.
db.getInvestorsFromValidatedInvests(function(err,result){
if(err){console.log(err);}
console.log(result);
List = result;
console.log('Inside the List scope'+List[0]);
for(var i=0;i<List.length;i++){
db.returnTotalInvestingfromValidatedInvests(List[i],function(err,rst){
if(err){console.log(err);}
console.log(rst);
var dailySaving = ((rst*multiplier)/100);
console.log('Daily Savings which is going to be added is that : ' +dailySaving);
console.log(List[i]);
db.multiplyInvestAndAddSavings(List[i],dailySaving,function(err,rest){
if(err){console.log(err);}
console.log(rest);
});
});
}
});
res.json('Going to be Prepared...');
});
Here i am sharing you each of these database functions :
First function gets the owner e-mails from db and returns an array and then, i want to do some operations with each elements of this array and i decided to use for loop but it does not work properly.
Db.prototype.getInvestorsFromValidatedInvests = function(callback){
MongoClient.connect("mongodb://xxxxxxx:xxxxxxx#ds9999.mlab.com:99986/xxxxxx", function (err, db) {
if (err) { return console.dir(err); }
db.collection('validatedInvests').distinct("owner",function(err,result){
if(err){console.log(err); callback(err,null);}
callback(null,result);
});
db.close();
});
}
Db.prototype.returnTotalInvestingfromValidatedInvests = function(owner,callback){
MongoClient.connect("mongodb://xxxxxxxx:xxxxxxx#ds09999.mlab.com:9996/aslanservices", function (err, db) {
if (err) { return console.dir(err); }
db.collection('validatedInvests').find({"owner":owner}).toArray(function(err,result){
var sum=0;
if(err){console.log(err); callback(err,null);}
for(var j=0;j<result.length;j++){
sum+=parseInt(result[j].amount);
}
callback(null,sum);
});
db.close();
});
}
Db.prototype.multiplyInvestAndAddSavings = function(owner,amount,callback){
MongoClient.connect("mongodb://xxxxxx:xxxxxxx#ds99996.mlab.com:39996/aslanservices", function (err, db) {
if (err) { return console.dir(err); }
console.log('Db talking amount is'+amount);
db.collection('investClients').updateOne({"email":owner},
{
$inc : {
"savingsAccount" : +amount
},
},
{
upsert : true
}
,function(err,result){
if(err){callback(err,null);}
callback(null,result);
});
db.close();
});
}
The output is :
Inside the List Scope > lets say blablabla#gmail.com
220
Daily Savings which is going to be added is that : 8.8
undefined
150
Daily Savings which is going to be added is that : 6
undefined
Db talking amount is 8.8
Db talking amount is 6
Expected ouput is that :
220
Daily Savings which is going to be added is that : 8.8
blablabla#gmail.com
Db talking amount is 8.8
150
Daily Savings which is going to be added is that : 6
secondblablabla#gmail.com
Db talking amount is 6
The problem is that you're using a for loop with async functionality inside. Let me try to explain:
for(var i=0;i<List.length;i++){
// first iteration, the variable i is
}
now the first async function gets executed:
db.returnTotalInvestingfromValidatedInvests(List[i],function(err,rst){
/*this is an async callback which will be executed in a
different context in a later time, when the response is received*/
}
So what happens is the main execution where the for loop is being executed doesn't wait for the response (since async) from the above function, and moves forward incrementing the variable i executing the next iteration with your function db.returnTotalInvestingfromValidatedInvests and it onwards.
Very soon the whole for loop is ended whether or not all or any of your responses have arrived.
The reason you're getting undefined is because at the end of the loop the variable i has value of List.length which means that it is accessing List[list.length]. Which will always be undefined since the max index is always length-1.
Summary: You will have to keep a track of the element at List[i] somehow so that when the final (nested) function's async callback is triggered, you can update the correct element. Some people do that by creating hashes, some do it by converting their logic to synchronous in terms of loop.
Advice:
Try logging out i in each of the function callbacks to see the variable being changed quickly.
Hope the explanation helps :)
As requested, here's one way to do it:
function onInvestsDone(err, obj){
if(err!=null){
// something went wrong
}
else{
// do anything here with your final response
}
}
for(var i=0;i<List.length;i++){
returnInvests(List, i, onInvestsDone);
}
function returnInvests(listArray, index, cb){
db.returnTotalInvestingfromValidatedInvests(listArray[index],function(err,rst){
if(err){
console.log(err);
cb(err); // call callback with error
}
console.log(rst);
var dailySaving = ((rst*multiplier)/100);
console.log('Daily Savings which is going to be added is that : ' +dailySaving);
console.log(listArray[index]);
db.multiplyInvestAndAddSavings(listArray[index],dailySaving,function(err,rest){
if(err){
console.log(err);
cb(err); // call callback with error
}
console.log(rest);
// successfully done everything
cb(null, rest);
});
});
}
I'm using svn-spawn library to update/commit files to svn. Problem is my app calls svn up/commit in a loop, and because of the async nature of the call, svn-up is called from the next iteration of the loop before the previous svn-up can finish.
How to handle this issue? Is there any way to prevent the next call from happening until the previous one is complete?
Figured out a way to do it using async module.
async.series can be used to execute async tasks in a serial fashion.
This is how I did it.
function commitFile(arg, callback) {
svnClient.getStatus(filePath, function(err, data) {
//...
svnClient.commit(['Commit msg', filePath], callback);
//...
});
}
var toCommit = [];
for (var i = 0, len = requests.length; i < len; i++) {
//Adding files to commit, async.apply enables adding arguments to the anonymous function
toCommit.push(async.apply(function(arg, cb) {
commitFile(arg, cb);
}, 'arg1'));
}
async.series(toCommit,function (err, result) {
console.log('Final callback');
if(err) {
console.log('error', err);
} else {
console.log('result of this run: ' + result);
}
});
async.series needs an array of functions which must call a callback once they are done. It uses the callback to determine that the current function in done executing and only then it will pick the next function to execute.
I wonder if it's possible to implement synchronous retrieval of smembers of redis with node_redis.
_.each(seeds, function(subseed, key, list){
client.smembers(subseed, function (err, replies) {
retrieved = retrieved.concat(replies)
})
})
client.quit();
console.log("start")
console.log(retrieved.length)
OUTPUT:
start
0
10
So it looks like I need somehow to get to the point when smembers finishes it's run, ot alternatively to run smembers in synchronous mode.
How can I solve this problem?
Why do you want to make it synchronous?
If you want to do something after members, just do it in callback function.
var callback=function(res){
retrived=retrieved.concat(res);
//continue do other things
};
client.smembers("offer", function (err, replies) {
if(!err){
callback(replies);
}
})
If you want to do something after a loop,you can try _.after of underscore.js,for example:
var times=10;
var arrayOfReplies=[]; //store all replies
var callback=function(){
//do something with arrayOfReplies
}
var afterAll = _.after(times,callback); //afterAll's callback is a function which will be run after be called 10 times
for(var i=0;i<10;i++){
client.smembers("offer", function (err, replies) {
if(!err){
arrayOfReplies=arrayOfReplies.concat(replies);
afterAll();
}
})
}
see more:http://underscorejs.org/#after
I know node.js run asynchronously, so outer functions execute earlier than the inner. But what is the way to access the notification array outside the for loop? I would like to access all the values in array at once, is this feasible?
var notification=[];
for(var j=0;j<6; j++)
{
getNotification(response[j].sender_id,function(results) // a function called
{
notification[j] =results;
console.log(notification); // output: correct
});
}
console.log(notification); // output: [], need notification array values here
EDIT: If you don't want to use third party libs, this is how to do this in your own code.
/* jshint node:true*/
function getNotifications(responses, callbackToMainProgramLogic) {
'use strict';
var results = [];
function getNotificationAsync(response) {
getNotification(response.sender_id, function (data) {
results.push(data);
if (responses.length) {
getNotificationAsync(responses.pop());//If there are still responses, launch another async getNotification.
} else {
callbackToMainProgramLogic(results);//IF there aren't we're done, and we return to main program flow
}
});
}
getNotificationAsync(responses.pop());
}
getNotifications(someArrayOfResonses, function (dataFromNotifications) {
console.log('The collected data: ' + JSON.stringify(dataFromNotifications, 0, 4));
});
If you absolutely must, you could do something ridiculous like this. Your logic in the loopUntilDatReceived would be waiting for array sizes, not waiting for a non-empty string, but the idea is similar, and you shouldn't be using this anyway! :)
var fileData = '';
fs.readFile('blah.js', function (err, data) { //Async operation, similar to your issue.
'use strict';
fileData = data;
console.log('The Data: ' + data);
});
function loopUntilDataReceived() {
'use strict';
process.nextTick(function () {//A straight while loop would block the event loop, so we do this once per loop around the event loop.
if (fileData === '') {
console.log('No Data Yet');
loopUntilDataReceived();
} else {
console.log('Finally: ' + fileData);
}
});
}
loopUntilDataReceived();
Did I mention this is ridiculous? Honestly, this is an awful idea, but it may help you understand what is going on and how the Node event loop works, and why what you want is not possible. AND why the other posts about callbacks, and flow control libraries are the way to go.
First off, you're having a closure issue in your code (please see the details here)
Then, you simply can't have the array values just next to the loop, because the values are not ready at this point.
You need to wait until all 6 of your getNotification calls get resolved. You can do that with the async library. Something like:
var notification = [];
function createRequest (index) {
return function (callback) {
getNotification(response[index].sender_id, function(results) {
notification[index] = results;
callback(results);
});
}
}
var requests = [];
for(var j=0;j<6; j++) {
requests.push(createRequest(j));
}
async.parallel(requests, function (allResults) {
// notifications array is ready at this point
// the data should also be available in the allResults array
console.log(notifications);
});
Send a callback to the notification loop like this:
var notification=[];
getNotificationArray( function() {
console.log(notification);
});
function getNotificationArray (callback)
{
for(var j=0;j<6; j++)
{
getNotification(response[j].sender_id,function(results) // a function called
{
notification[j] =results;
console.log(notification); // output: correct
});
}
callback();
}
I am trying to allow users to export their contact list in csv format. I am confused on how to run export_connect_csv() function. should i put it in child process or process.nextTick?
function export_connect_csv(user_id, file_location){
mysqlPool.getConnection(function(err, connection){
var csv_row = "Email,First Name,Last Name,Status,Created\n";
function processRow (row) {
var csv_row = row.email+','+row.first_name+','+row.last_name+','+row.status+','+row.created+"\n";
fs.appendFile(file_location, csv_row, function (err) {
if(err){
throw err;
}
});
}
fs.appendFile(file_location, csv_row, function (err) {
if(err){
throw err;
}
var query = connection.query('SELECT * FROM contacts where user_id = "'+user_id+'"');
query
.on('error', function(err) {
//handle error
})
.on('fields', function(fields) {
})
.on('result', function(row) {
processRow(row);
})
.on('end', function() {
//email now
console.log('done');
});
});
});
}
var exportContacts = function(req, res){
var user_id = req.params.user_id || 0;
export_connect_csv(user_id);
res.json({});
};
You don't need to use either, you can just call the function. All of that code will run assynchronously, both getConnection and fs.appendFile. However, you will run into a conflict in the case two users try to export at the same time. You have the following options:
1) You pass a unique file_name every time you call that function
2) You keep things exactly as they are and use fs.appendFileSync to make sure they don't overlap each other but that would block you
3) Or probably the best solution is do what you intended to do with the Process.nextTick, but instead you should use setImmediate and appendFileSync to be able to synchronize writes from several users simultaneously (write only a row at a time to avoid blocking for long periods):
setImmediate(function () {
fs.appendFileSync('filename', JUST_A_SINGLE_ROWW)
});
This is because a recursive process.nextTick can starve the event loop and effecively block you (hence the use of setImmediate) and you need to use fs.appendFileSync because two users might write to the same file simultaneously.
More on setImmediate vs nextTick:
setImmediate vs. nextTick
More info on appendFile: http://nodejs.org/api/fs.html#fs_fs_appendfile_filename_data_options_callback