Basically, I want to pit two asynchronous calls against each other, and only use the winner.
I cannot seem to figure out how to do this, only how to prevent it. Is it remotely possible?
Lame pseudo-code:
//rigging a race
function MysqlUser()
{
setTimeout(function(){
return "mysqluser";
}, 500);
}
function ADUser()
{
setTimeout(function(){
return "aduser";
}, 1000);
}
function getUser()
{
var user = null;
user = ADBind();
user = MysqlBind();
//if user != null
return user;
//else?
}
I'd like (in this instance) for MysqlUser to win out over ADUser.
Any help would be appreciated.
You can write a simple first function that takes a list of task and calls back with the result of only the first to complete:
function first(tasks, cb) {
var done = false;
tasks.forEach(function(task) {
task(function(result) {
if(done) return;
done = true;
cb(result);
});
});
}
Then:
function mysqlUser(cb) {
setTimeout(function() {
cb("mysqluser");
}, 500);
}
function adUser(cb) {
setTimeout(function() {
cb("aduser");
}, 1000);
}
first([mysqlUser, adUser], function(user) {
console.log(user);
});
This might need some more thought if you want to cope with both operations failing.
Related
I'm learning node and would like to optimize the code I did. I tried using Async.parallel to perform the operations and when finished return a json.
I am new to js node and I'm trying to do with async.parallel but I returned [Function] in other code that I'm trying to understand it
getTabletIntRout: function(req, res) {
var reqMAC = req.param('id_tablet');
Tablet.findOne(reqMAC).populate('rout_tablet').exec(function(err, tablet) {
if (err) return next(err);
if (!tablet) return next();
var arrRoutes = tablet.rout_tablet;
if (arrRoutes.length > 0) {
var routesNotRemoved = [];
arrRoutes.forEach(function(route) {
if (route.removed == 'no') {
Rout.findOne(route.id)
.populate('rout_assigned') // Pin
.populate('in_rout') // Tablet
.populate('rout_description_assigned')
.exec(function(err, rout2) {
var arrRout = rout2.rout_assigned;
var routsNotRemoved = [];
if (arrRout.length > 0) {
arrRout.forEach(function(ruta) {
if (ruta.removed == 'no') {
routsNotRemoved.push(ruta);
}
});
}
var arrTablets = rout2.in_rout;
var tabletsNotRemoved = [];
if (arrTablets.length > 0) {
arrTablets.forEach(function(tab) {
if (tab.removed == 'no') {
tabletsNotRemoved.push(tab);
}
});
}
var arrDesc = rout2.rout_description_assigned;
var descripNotRemoved = [];
if (arrDesc.length > 0) {
arrDesc.forEach(function(desc) {
if (desc.removed == 'no') {
descripNotRemoved.push(desc);
}
});
}
rout2.rout_assigned = routsNotRemoved;
rout2.in_rout = tabletsNotRemoved;
rout2.rout_description_assigned = descripNotRemoved;
routesNotRemoved.push(rout2);
});
}
});
setTimeout(function() {
if (routesNotRemoved.length > 0) {
res.json({ info: routesNotRemoved });
} else {
return res.json({"error": "-1", "message": "Todas las rutas asociadas a esa tablet están eliminadas"});
}
}, 2000);
} else {
return res.json({"error": "-2", "message": "No existen rutas asociadas en esa tablet"});
}
}););});}},
I will try to provide some thoughts, hopefully some will make sense in your domain.
Extract a function to make people understand what you're doing in that big function
So instead of
Tablet.findOne(reqMAC).populate('rout_tablet').exec(function(err, tablet) { // ...
You would have
Tablet.findOne(reqMAC).populate('rout_tablet').exec(meaningfulFunctionName);
Don't repeat yourself
So your code becomes shorter and whenever the reader of your code finds a function name he / she already knows what is happening inside that
if (arrRout.length > 0) {
arrRout.forEach(function(ruta) {
if (ruta.removed == 'no') {
routsNotRemoved.push(ruta);
}
});
}
No need to check for empty arrRout as the argument function to arrRout.forEach will simply not run in the case of length being zero.
What you wrote is just a filter function, so why not using filter? Like so
arrRout.filter(function(ruta) {
return ruta.removed == 'no';
});
You can also reuse this, if you extract the anonymous function, for arrTablets and arrDesc.
On the argument: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/filter
Don't use a huge if else
Either check for the inverse or return a default or something that makes sense in your domain, but don't have that big chunk of logic, it makes it hard to reason about your code.
Extract more function so that it's easier to use async
You might want to have something like this
async.waterfall([
function(next) {
// here you can put
// Tablet.findOne(reqMAC).populate('rout_tablet').exec
// invoke next with err, tablet
},
function(tablet, next) {
async.each(arrRoutes, function(arrRoute, nextEach) {
// write your code logic here
});
}
], function() {
// decide what to invoke res.json with
});
Remember to extract functions after you're done putting the logic inside the async steps, I didn't do it so it is more clear where to put what.
I hope this makes sense, feel free to ask if you have any doubts.
Next time you post a question please make sure to properly indent it, don't just paste it here.
I'm trying to convert an existing API to work with RxJS... fairly new to node, and very new to RxJs, so please bear with me.
I have an existing API (getNextMessage), that either blocks (asynchronously), or returns a new item or error via a node-style (err, val) callback, when the something becomes available.
so it looks something like:
getNextMessage(nodeStyleCompletionCallback);
You could think of getNextMessage like an http request, that completes in the future, when the server responds, but you do need to call getNextMessage again, once a message is received, to keep getting new items from the server.
So, in order to make it into an observable collection, I have to get RxJs to keep calling my getNextMessage function until the subscriber is disposed();
Basically, I'm trying to create my own RxJs observable collection.
The problems are:
I don't know how to make subscriber.dispose() kill the async.forever
I probably shouldn't be using async.forever in the first place
I'm not sure I should be even getting 'completed' for each message - shouldn't that be at the end of a sequence
I'd like to eventually remove the need for using fromNodeCallback, to have a first class RxJS observable
Clearly I'm a little confused.
Would love a bit of help, thanks!
Here is my existing code:
var Rx = require('rx');
var port = require('../lib/port');
var async = require('async');
function observableReceive(portName)
{
var observerCallback;
var listenPort = new port(portName);
var disposed = false;
var asyncReceive = function(asyncCallback)
{
listenPort.getNextMessage(
function(error, json)
{
observerCallback(error, json);
if (!disposed)
setImmediate(asyncCallback);
}
);
}
return function(outerCallback)
{
observerCallback = outerCallback;
async.forever(asyncReceive);
}
}
var receive = Rx.Observable.fromNodeCallback(observableReceive('rxtest'));
var source = receive();
var subscription = source.forEach(
function (json)
{
console.log('receive completed: ' + JSON.stringify(json));
},
function (error) {
console.log("receive failed: " + error.toString());
},
function () {
console.log('Completed');
subscription.dispose();
}
);
So here's probably what I would do.
var Rx = require('Rx');
// This is just for kicks. You have your own getNextMessage to use. ;)
var getNextMessage = (function(){
var i = 1;
return function (callback) {
setTimeout(function () {
if (i > 10) {
callback("lawdy lawd it's ova' ten, ya'll.");
} else {
callback(undefined, i++);
}
}, 5);
};
}());
// This just makes an observable version of getNextMessage.
var nextMessageAsObservable = Rx.Observable.create(function (o) {
getNextMessage(function (err, val) {
if (err) {
o.onError(err);
} else {
o.onNext(val);
o.onCompleted();
}
});
});
// This repeats the call to getNextMessage as many times (11) as you want.
// "take" will cancel the subscription after receiving 11 items.
nextMessageAsObservable
.repeat()
.take(11)
.subscribe(
function (x) { console.log('next', x); },
function (err) { console.log('error', err); },
function () { console.log('done'); }
);
I realize this is over a year old, but I think a better solution for this would be to make use of recursive scheduling instead:
Rx.Observable.forever = function(next, scheduler) {
scheduler = scheduler || Rx.Scheduler.default,
//Internally wrap the the callback into an observable
next = Rx.Observable.fromNodeCallback(next);
return Rx.Observable.create(function(observer) {
var disposable = new Rx.SingleAssignmentDisposable(),
hasState = false;
disposable.setDisposable(scheduler.scheduleRecursiveWithState(null,
function(state, self) {
hasState && observer.onNext(state);
hasState = false;
next().subscribe(function(x){
hasState = true;
self(x);
}, observer.onError.bind(observer));
}));
return disposable;
});
};
The idea here is that you can schedule new items once the previous one has completed. You call next() which invokes the passed in method and when it returns a value, you schedule the next item for invocation.
You can then use it like so:
Rx.Observable.forever(getNextMessage)
.take(11)
.subscribe(function(message) {
console.log(message);
});
See a working example here
I have asynchronous query in Node.js. Variable sq3 is a connection variable.
For example something like this:
for (var i in res) {
if (i == 1) {
sq3.query("SELECT * from students;",
function (err, res) {
if (err) {
throw err;
} else {
if (res.length == 1) {
//do something
} else {
//break for
}
}
});
sq3.end();
}
}
How can I break from callback function?
Thanks
Just do it like this, using recursion instead of loops. Not only does this allow you to achieve the logic you want. It also doesn't spin up a bunch of async requests at once. They execute in turn, but asynchronously, so it's still performant.
function lookatEntireResponse(res) {
function lookAtItemInResponse(item) {
if(item == 1) {
sq3.query("SELECT * from students;",
function(err, res) {
if (err)
{
throw err;
}
else
{
if(res.length==1)
{
doSomething(item);
lookAtItemInResponse(res.shift());
}
else
{
//just don't call the next lookAtItemInResponse function, effectively same thing as "break;"
}
}
});
sq3.end();
} else {
lookAtItemInResponse(res.shift());
}
}
lookAtItemInResponse(res.shift());
}
You can consider throttling simultaneous requests with similar logic (say allowing 10 such requests per lookAtItem call. This way you can achieve a hybrid of the two methods, and then just optimize the number of simultaneous requests for performance. The Async library makes stuff like this easier.
In your code fragment, you can't break from the within the callback function. In node.js, callback functions run at an unspecified later time on the same thread. This means but the time you callback function executes, the for loop has long since finished.
To get the effect you want, you need to restructure you code quite significantly. Here's an example of how you could do it (untested!!). The idea is to keep calling doSomething() with the list of items, shrinking it by one element each time until the desired result is achieved (your break condition).
function doSomething(res)
{
while (res.length > 0)
{
i = res.shift(); // remove the first element from the array and return it
if (i == 1)
{
sq3.query("SELECT * from students;",
function(err, res) {
if (err)
{
throw err;
}
if (res.length==1)
{
//do something
// continue with the remaining elements of the list
// the list will shrink by one each time as we shift off the first element
doSomething(res);
}
else
{
// no need to break, just don't schedule any more queries and nothing else will be run
}
});
sq3.end();
break; // break from the loop BEFORE the query executes. We have scheduled a callback to run when the query completes.
}
}
}
for (var i = 0; i < res.length; i++) {
if (i == 1) {
sq3.query("SELECT * from students;",
function (err, res) {
if (err) {
throw err;
} else {
if (res.length == 1) {
//do something
} else {
i = res.length
}
}
});
sq3.end();
}
}
Using mongoskin, I can do a query like this, which will return a cursor:
myCollection.find({}, function(err, resultCursor) {
resultCursor.each(function(err, result) {
}
}
However, I'd like to call some async functions for each document, and only move on to the next item on the cursor after this has called back (similar to the eachSeries structure in the async.js module). E.g:
myCollection.find({}, function(err, resultCursor) {
resultCursor.each(function(err, result) {
externalAsyncFunction(result, function(err) {
//externalAsyncFunction completed - now want to move to next doc
});
}
}
How could I do this?
Thanks
UPDATE:
I don't wan't to use toArray() as this is a large batch operation, and the results might not fit in memory in one go.
A more modern approach that uses async/await:
const cursor = db.collection("foo").find({});
while(await cursor.hasNext()) {
const doc = await cursor.next();
// process doc here
}
Notes:
This may be even more simple to do when async iterators arrive.
You'll probably want to add try/catch for error checking.
The containing function should be async or the code should be wrapped in (async function() { ... })() since it uses await.
If you want, add await new Promise(resolve => setTimeout(resolve, 1000)); (pause for 1 second) at the end of the while loop to show that it does process docs one after the other.
If you don't want to load all of the results into memory using toArray, you can iterate using the cursor with something like the following.
myCollection.find({}, function(err, resultCursor) {
function processItem(err, item) {
if(item === null) {
return; // All done!
}
externalAsyncFunction(item, function(err) {
resultCursor.nextObject(processItem);
});
}
resultCursor.nextObject(processItem);
}
since node.js v10.3 you can use async iterator
const cursor = db.collection('foo').find({});
for await (const doc of cursor) {
// do your thing
// you can even use `await myAsyncOperation()` here
}
Jake Archibald wrote a great blog post about async iterators, that I came to know after reading #user993683's answer.
This works with large dataset by using setImmediate:
var cursor = collection.find({filter...}).cursor();
cursor.nextObject(function fn(err, item) {
if (err || !item) return;
setImmediate(fnAction, item, arg1, arg2, function() {
cursor.nextObject(fn);
});
});
function fnAction(item, arg1, arg2, callback) {
// Here you can do whatever you want to do with your item.
return callback();
}
If someone is looking for a Promise way of doing this (as opposed to using callbacks of nextObject), here it is. I am using Node v4.2.2 and mongo driver v2.1.7. This is kind of an asyncSeries version of Cursor.forEach():
function forEachSeries(cursor, iterator) {
return new Promise(function(resolve, reject) {
var count = 0;
function processDoc(doc) {
if (doc != null) {
count++;
return iterator(doc).then(function() {
return cursor.next().then(processDoc);
});
} else {
resolve(count);
}
}
cursor.next().then(processDoc);
});
}
To use this, pass the cursor and an iterator that operates on each document asynchronously (like you would for Cursor.forEach). The iterator needs to return a promise, like most mongodb native driver functions do.
Say, you want to update all documents in the collection test. This is how you would do it:
var theDb;
MongoClient.connect(dbUrl).then(function(db) {
theDb = db; // save it, we'll need to close the connection when done.
var cur = db.collection('test').find();
return forEachSeries(cur, function(doc) { // this is the iterator
return db.collection('test').updateOne(
{_id: doc._id},
{$set: {updated: true}} // or whatever else you need to change
);
// updateOne returns a promise, if not supplied a callback. Just return it.
});
})
.then(function(count) {
console.log("All Done. Processed", count, "records");
theDb.close();
})
You can do something like this using the async lib. The key point here is to check if the current doc is null. If it is, it means you are finished.
async.series([
function (cb) {
cursor.each(function (err, doc) {
if (err) {
cb(err);
} else if (doc === null) {
cb();
} else {
console.log(doc);
array.push(doc);
}
});
}
], function (err) {
callback(err, array);
});
You could use a Future:
myCollection.find({}, function(err, resultCursor) {
resultCursor.count(Meteor.bindEnvironment(function(err,count){
for(var i=0;i<count;i++)
{
var itemFuture=new Future();
resultCursor.nextObject(function(err,item)){
itemFuture.result(item);
}
var item=itemFuture.wait();
//do what you want with the item,
//and continue with the loop if so
}
}));
});
You can get the result in an Array and iterate using a recursive function, something like this.
myCollection.find({}).toArray(function (err, items) {
var count = items.length;
var fn = function () {
externalAsyncFuntion(items[count], function () {
count -= 1;
if (count) fn();
})
}
fn();
});
Edit:
This is only applicable for small datasets, for larger one's you should use cursors as mentioned in other answers.
A more modern approach that uses for await:
const cursor = db.collection("foo").find({});
for await(const doc of cursor) {
// process doc here with await
await processDoc(doc);
}
You could use simple setTimeOut's. This is an example in typescript running on nodejs (I am using promises via the 'when' module but it can be done without them as well):
import mongodb = require("mongodb");
var dbServer = new mongodb.Server('localhost', 27017, {auto_reconnect: true}, {});
var db = new mongodb.Db('myDb', dbServer);
var util = require('util');
var when = require('when'); //npm install when
var dbDefer = when.defer();
db.open(function() {
console.log('db opened...');
dbDefer.resolve(db);
});
dbDefer.promise.then(function(db : mongodb.Db){
db.collection('myCollection', function (error, dataCol){
if(error) {
console.error(error); return;
}
var doneReading = when.defer();
var processOneRecordAsync = function(record) : When.Promise{
var result = when.defer();
setTimeout (function() {
//simulate a variable-length operation
console.log(util.inspect(record));
result.resolve('record processed');
}, Math.random()*5);
return result.promise;
}
var runCursor = function (cursor : MongoCursor){
cursor.next(function(error : any, record : any){
if (error){
console.log('an error occurred: ' + error);
return;
}
if (record){
processOneRecordAsync(record).then(function(r){
setTimeout(function() {runCursor(cursor)}, 1);
});
}
else{
//cursor up
doneReading.resolve('done reading data.');
}
});
}
dataCol.find({}, function(error, cursor : MongoCursor){
if (!error)
{
setTimeout(function() {runCursor(cursor)}, 1);
}
});
doneReading.promise.then(function(message : string){
//message='done reading data'
console.log(message);
});
});
});
Check below algorithm...
users = getAllUsers();
for(i=0;i<users.length;i++)
{
contacts = getContactsOfUser(users[i].userId);
contactslength = contacts.length;
for(j=o;j<contactsLength;j++)
{
phones = getPhonesOfContacts(contacts[j].contactId);
contacts[j].phones = phones;
}
users[i].contacts = contacts;
}
return users;
I want to develop such same logic using node.js.
I have tried using async with foreach and concat and foreachseries functions. But all fail in the second level.
While pointer is getting contacts of one user, a value of i increases and the process is getting started for next users.
It is not waiting for the process of getting contacts & phones to complete for one user. and only after that starting the next user. I want to achieve this.
Actually, I want to get the users to object with proper
Means all the sequences are getting ruined, can anyone give me general idea how can I achieve such a series process. I am open to change my algorithm also.
In node.js you need to use asynchronous way. Your code should look something like:
var processUsesrs = function(callback) {
getAllUsers(function(err, users) {
async.forEach(users, function(user, callback) {
getContactsOfUser(users.userId, function(err, contacts) {
async.forEach(contacts, function(contact, callback) {
getPhonesOfContacts(contacts.contactId, function(err, phones) {
contact.phones = phones;
callback();
});
}, function(err) {
// All contacts are processed
user.contacts = contacts;
callback();
});
});
}, function(err) {
// All users are processed
// Here the finished result
callback(undefined, users);
});
});
};
processUsers(function(err, users) {
// users here
});
You could try this method without using async:
function getAllUserContacts(users, callback){
var index = 0;
var results = [];
var getUserContacts = function(){
getContactsOfUser(users[index].userId, function(contacts){
var index2 = 0;
var getContactsPhones = function(){
getPhonesOfContacts(contacts[index2].contactId, function(phones){
contacts[index2].phones = phones;
if(index2 === (contacts.length - 1)){
users[index].contacts = contacts;
if(index === (users.length - 1)){
callback(users)
} else {
index++;
getUserContacts();
}
}else{
index2++;
getContactsPhones();
}
});
}
getContactsPhones();
});
}
getUserContacts();
}
//calling the function
getAllUsers(function(users){
getAllUsersWithTheirContacts(users, function(usersWithContacts){
console.log(usersWithContacts);
})
})
//Asynchronous nested loop
async.eachSeries(allContact,function(item, cb){
async.eachSeries(item,function(secondItem,secondCb){
console.log(secondItem);
return secondCb();
}
return cb();
},function(){
console.log('after all process message');
});