I wonder if it's possible to implement synchronous retrieval of smembers of redis with node_redis.
_.each(seeds, function(subseed, key, list){
client.smembers(subseed, function (err, replies) {
retrieved = retrieved.concat(replies)
})
})
client.quit();
console.log("start")
console.log(retrieved.length)
OUTPUT:
start
0
10
So it looks like I need somehow to get to the point when smembers finishes it's run, ot alternatively to run smembers in synchronous mode.
How can I solve this problem?
Why do you want to make it synchronous?
If you want to do something after members, just do it in callback function.
var callback=function(res){
retrived=retrieved.concat(res);
//continue do other things
};
client.smembers("offer", function (err, replies) {
if(!err){
callback(replies);
}
})
If you want to do something after a loop,you can try _.after of underscore.js,for example:
var times=10;
var arrayOfReplies=[]; //store all replies
var callback=function(){
//do something with arrayOfReplies
}
var afterAll = _.after(times,callback); //afterAll's callback is a function which will be run after be called 10 times
for(var i=0;i<10;i++){
client.smembers("offer", function (err, replies) {
if(!err){
arrayOfReplies=arrayOfReplies.concat(replies);
afterAll();
}
})
}
see more:http://underscorejs.org/#after
Related
How do I get data from a query OUTSIDE the query in Node.JS?
I can get results printed to the console INSIDE the function but I can't get the data OUTSIDE to use in other places in my application.
this.getMyQuestion = function(id) {
var query = connection.query('select * from questions where id = ' + connection.escape(id), function(err, result) {
if(err) {
console.error(err);
return;
}
//console.log(result[0].question); //displays in console
return(result[0].question);
});
}
var test = this.getMyQuestion(1);
console.log(test) //returns undefined.
You're using an asynchronous function, so by the time you run the console.log(test) after calling your function, it hasn't finished running and hasn't returned anything yet.
That's what callbacks are for. You pass a callback to your function, and when it finishes execution, it calls that callback function instead of returning a value like it is now.
For example :
this.getMyQuestion = function(id, callback) {
var query = connection.query('select * from questions where id = ' + connection.escape(id), function(err, result) {
callback(null, result[0].question);
});
}
this.getMyQuestion(1, function(err, question){
// Do what you want
});
You can also promises to get data from async functions.
I wish to update a Mongo collection in some code that looks like this:
var Q = Npm.require('q');
var db = new Mongo.Collection('mydb');
function doSomething() {
var d = Q.defer();
setTimeout( function() {
d.resolve();
}, 1000);
return d.promise;
}
doSomething().then( function() {
console.log('before find');
var records = db.find({}).fetch(); // blocking operation never completes
console.log('after find');
console.log(records); // should be []
});
When running meteor with the above code, it will get as far as logging "before find" but then the execution is halted waiting for db.find to complete. It never completes.
Are there any solutions or workarounds for this?
Update: it seems to be the .fetch() that causes the issue. I need this part though, I want to manipulate the data I am receiving from Mongo.
Instead of using fetch, add a callback function.
It will allow you to manipulate the data after its retrieved:
var records = db.find({}, function(error, data){
// Do something with your data here
});
** Edit - with the callback above, a curser is returned. If you want to return an Array with the results, use the following:
var records = db.find({}).toArray(function(error, data){
// Do something with your data here
});
i have a sample function in which i want to trigger 3 sql queries in one express post request
function getRelatedSalespersonByCardcode(req, res) {
var reqJson = JSON.parse(req.body.json);
var count = Object.keys(reqJson.cardcode).length;
var tmpResult1, tmpResult2, tmpResult3 = [];
var q = sql.open(connstr, function (err) {
if (err) {
console.log(err);
return;
}
for (var i = 0; i < count; i++) {
q.queryRaw("SELECT Division, Salesperson FROM SomeDB.dbo.MS2_Rel_BusinessPartnerSalesperson WHERE CardCode = " + reqJson.cardcode[i], function (e, results) {
if (e) {
console.log(e);
return;
}
tmpResult1.push(results);
});
q.queryRaw("SELECT SlpCode, SlpName, Memo, Commission, GroupCode, UserSign, Active, U_wpABIS, U_sweDW," +
" U_sweATT, U_sweDIV, U_sweEMPLOYEE, U_sweRETAILER FROM SomeDB.dbo.OSLP WHERE U_sweId = " + tmpResult1[1], function (e, results) {
if (e) {
console.log(e);
return;
}
tmpResult2.push(results);
});
q.queryRaw("SELECT Code, Name, U_sweSALES FROM SomeDB.dbo.[#SWEDIV] WHERE Code = " + tmpResult3[0], function (e, results) {
if (e) {
console.log(e);
return;
}
tmpResult3.push(results);
});
}
});
res.send(200, tmpResult2);
}
Anyway after 1 function is called inside my req, res function the callback is triggert.. so it jumps directly to the res.send(...) line.
i've played a bit around and it seems that this is how express works.
after a bit of googleing around i found out that i have to use the async lib.
i'd like to ask why express is working like this and if anybody maybe have a better solution than the async approach. i simply need a way to realize my scenario.
This is no issue of express and no issue at all. This is like node.js async programming is working. Let's examine what's happening in your code:
var tmpResult2;
// 1. async open sql connection
var q = sql.open(connstr, function (err) {
// 3. query sql connection
});
// 2. render result
res.send(200, tmpResult2);
SQL connection is opened asynchronously.
Express render is called.
SQL queries are executed.
As a consequence this code piece sends data to the client before the data was fetched. So the most simple solution is to invoke res.send inside of the callback like this:
var tmpResult2;
// 1. async open sql connection
var q = sql.open(connstr, function (err) {
// 2. render result
res.send(200, tmpResult2);
});
You can use the wonderful async module to deal with the queries (Take a look at the async waterfall function).
I have folowing script
var email_list = ['email1#email.com', 'email2#email.com',....'email100#email.com'];
for(i=0;i<email_list.length;i++){
if(checkEmail(email_list[i])){
//do processing save in db and email to email addresses.
}
}
This code will be blocking in nodejs how to make this non blocking?
You can do this without blocking the event loop at all, by using a recursive loop. This way what you end up with is only launching one database worker per call, at a give time. Assuming the database work you were doing was asynchronous, your code didn't really block the event loop. But the foor loop still launched a bunch of workers simultaneously, which will tend to clog the event loop(not block it). And you are right in that it is blocking the event loop while your for loop is counting from 0, to whatever the size of your array is. The following does exactly the same thing, but you only launch one database worker at a time(good), and you never count from 0 to length. Each worker is popped off the list after the work on the current email is done, and your global event loop is left to process other things, not email_list.length database requests simultaneously.
var email_list = ['email1#email.com', 'email2#email.com', 'email100#email.com'];
function checkEmailList(emails, emailCallBack, completionCallback) {
var someDataCollectdOverAllEmails = '';
function checkEmailAsync(email) {
db.doSomeDBWorkAsync(email, function (data) {
someDataCollectdOverAllEmails += data;
if (email_list.length) {
checkEmail(email_list.pop()); //If there are still emails to be checked, check the next one ine line
} else {
completionCallback(someDataCollectdOverAllEmails);//IF not, call the completionCallBack
}
emailCallBack(data);
});
}
checkEmailAsync(emails.pop());
}
function logIndividualEmailData(data) {
console.log('Sningle Email: ' + data);
}
function logGlobalEmailData(data) {
console.log('All Email Data: ' + data);
}
checkEmailList(email_list, logIndividualEmailData, logGlobalEmailData);
Process.nextTick example
process.nextTick(function () {
'use strict';
console.log('printed second');
while (true);
});
process.nextTick(function () {
'use strict';
console.log('never printed');
});
console.log('printed first');
Note however that in the example below, despite the fact that loopForever will run forever, it still allows both of our files to be read out. If we just had while(true) it would of course block and not allow this and one of our files data would not be printed out.
var files = ['blah.js', 'file.js'];
for(var i = 0; i < files.length; i++) {
fs.readFile(files[i], function (err, data) {
console.log('File data' + data);
function loopForver(loop) {//asynchronously loop forever, pretty cool, but only useful for really specific situations!
process.nextTick(function () {
if(loop) {
console.log('looping');
loopForver(true);
}
});
}
loopForver(true);
});
}
If I need to do stuff after the emails all send, I use the async library (docs), which provides some useful functions for control flow.
You will still need to rewrite checkEmail(email) into checkEmail(email, callback) as #S.D. suggests. In checkEmail you will want to call callback after everything is completed. This probably means that you will nest callbacks, calling the second async thing (sending the email) only after the first (db query) has completed successfully.
I also suggest that you follow convention by using the first callback argument as an err parameter. If you callback(null) you are explicitly saying 'there was no error'. #S.D.'s solution suggests instead callback(ok) which is the opposite of convention.
Here is an example showing a couple nested asynchronous functions and the async library.
edit - use async.eachLimit instead of async.each so you don't execute all 100 calls simultaneously
(function main(){
var emails = ["a#b", "c#d"];
var async = require('async');
async.eachLimit(
emails // array to iterate across
,10 // max simultaneous iterations
,checkEmail // an asynchronous iterator function
,function(err){ // executed on any error or every item successful
console.log('Callback of async.eachLimit');
if(err){
console.log('Error: '+err)
} else {
console.log('All emails succeeded');
};
}
);
console.log('Code below the async.eachLimit call will continue executing after starting the asynchronous jobs');
})();
function checkEmail(email, callback){
fetchFromDb(email, function(err, obj){
if(err){ return callback(err) };
sendEmail(email, function(err, obj){
if(err){ return callback(err)};
console.log('Both fetchFromDb and sendEmail have completed successfully for '+email);
callback(null);
});
});
};
function fetchFromDb(email, callback){
process.nextTick(function(){ // placeholder, insert real async function here
callback(null);
});
};
function checkEmail(email, callback){
process.nextTick(function(){ // placeholder, insert real async function here
callback(null);
});
};
I am trying to allow users to export their contact list in csv format. I am confused on how to run export_connect_csv() function. should i put it in child process or process.nextTick?
function export_connect_csv(user_id, file_location){
mysqlPool.getConnection(function(err, connection){
var csv_row = "Email,First Name,Last Name,Status,Created\n";
function processRow (row) {
var csv_row = row.email+','+row.first_name+','+row.last_name+','+row.status+','+row.created+"\n";
fs.appendFile(file_location, csv_row, function (err) {
if(err){
throw err;
}
});
}
fs.appendFile(file_location, csv_row, function (err) {
if(err){
throw err;
}
var query = connection.query('SELECT * FROM contacts where user_id = "'+user_id+'"');
query
.on('error', function(err) {
//handle error
})
.on('fields', function(fields) {
})
.on('result', function(row) {
processRow(row);
})
.on('end', function() {
//email now
console.log('done');
});
});
});
}
var exportContacts = function(req, res){
var user_id = req.params.user_id || 0;
export_connect_csv(user_id);
res.json({});
};
You don't need to use either, you can just call the function. All of that code will run assynchronously, both getConnection and fs.appendFile. However, you will run into a conflict in the case two users try to export at the same time. You have the following options:
1) You pass a unique file_name every time you call that function
2) You keep things exactly as they are and use fs.appendFileSync to make sure they don't overlap each other but that would block you
3) Or probably the best solution is do what you intended to do with the Process.nextTick, but instead you should use setImmediate and appendFileSync to be able to synchronize writes from several users simultaneously (write only a row at a time to avoid blocking for long periods):
setImmediate(function () {
fs.appendFileSync('filename', JUST_A_SINGLE_ROWW)
});
This is because a recursive process.nextTick can starve the event loop and effecively block you (hence the use of setImmediate) and you need to use fs.appendFileSync because two users might write to the same file simultaneously.
More on setImmediate vs nextTick:
setImmediate vs. nextTick
More info on appendFile: http://nodejs.org/api/fs.html#fs_fs_appendfile_filename_data_options_callback