Doing Something After Asynchronous Calls in Node - node.js

Suppose I have four distinct asynchronous operations that need to be run, and they can all be run independently. But there's one remaining function that needs to use all the data those asynchronous calls collect, so it can only be done once all of them are finished.
A simple way to do this is to make the asynchronous calls call each other one right after the other, and then finally call the final function, like so:
myObj.async1(function () {
myObj.async2(function () {
myObj.async3(function () {
myObj.async4(function () {
...
finalFunction();
But this is a poor way to do it, since node is built around asynchronous functionality for a reason. So instead, let's say we want to do:
myObj.async1(async1Callback);
myObj.async2(async2Callback);
myObj.async3(async3Callback);
myObj.async4(async4Callback);
if( //Some logic here to determine when all four functions have completed
finalFunction();
What's the best way to determine that logic? I considered having each function set a boolean variable to indicate whether it has completed, and then having a time-based emitter that constantly checks if all four variables are set to true and then calls finalFunction if they are, but that can get messy with having all those variables lying around.
Any thoughts on what's the best way to do this?

I would make use of the async library for this, e.g.
async.parallel([
myObj.async1,
myObj.async2,
myObj.async3,
myObj.async4
], function(err) {
if (err) throw err;
// Run final function now that all prerequisites are finished
finalFunction();
});
This assumes that each myObj.async* function takes a callback function as its only parameter and that callback's first parameter is an err param. For more info see the docs for async#parallel().

As #jabclab recommended, take a look at async as it manages much of the complexity for you. However, if you want to do something like this yourself here are a couple of alternatives.
Starting with a myObj that looks like:
var myObj = {
async1: function async1(cb) { setTimeout(function() {
console.log('async1');
cb(null, {name: 'async1'});
}, 1000)},
async2: function async2(cb) { setTimeout(function() {
console.log('async2');
cb(null, {name: 'async2'});
}, 500)},
async3: function async3(cb) { setTimeout(function() {
console.log('async3');
cb(null, {name: 'async3'});
}, 1001)},
async4: function async4(cb) { setTimeout(function() {
console.log('async4');
cb(null, {name: 'async4'});
}, 200)}
}
This version is hardcoded to call four specific functions and callback when the results are complete. The results passed back in an array ordered by completion. Each result object contains the name of the function as well as any error or success result.
function doFourSpecificThings(callback) {
var results = [];
var storeResults = function(fnName, err, resp) {
results.push( { fnName: fnName, err: err, resp: resp } );
if(results.length === 4 && callback) {
callback(results);
}
}
// Bind the callback to myObj and pass the name of the called function
// as the first argument
myObj.async1(storeResults.bind(myObj, 'async1'));
myObj.async2(storeResults.bind(myObj, 'async2'));
myObj.async3(storeResults.bind(myObj, 'async3'));
myObj.async4(storeResults.bind(myObj, 'async4'));
}
doFourSpecificThings(function(results) {
console.log(results);
});
Output:
async4
async2
async1
async3
Results:
[ { fnName: 'async4', err: null, resp: { name: 'async4' } },
{ fnName: 'async2', err: null, resp: { name: 'async2' } },
{ fnName: 'async1', err: null, resp: { name: 'async1' } },
{ fnName: 'async3', err: null, resp: { name: 'async3' } } ]
This version is a bit more flexible. The tasks are passed in as an array with the results being stored in the same order in the resulting array:
function doABunchOfStuff(tasks, callback) {
var results = [];
var expected = tasks.length;
var storeResults = function(idx, err, resp) {
results[idx] = { err: err, resp: resp };
--expected;
if((expected === 0) && callback) {
callback(results);
}
}
// Using bind here to pass the current index to the storeResults()
// callback as the first parameter
for(var i = 0; i < tasks.length; ++i) {
tasks[i](storeResults.bind(tasks[i], i));
}
}
doABunchOfStuff([
myObj.async1.bind(myObj),
myObj.async2.bind(myObj),
myObj.async3.bind(myObj),
myObj.async4.bind(myObj)],
function(results) {
console.log('\nResults:');
console.log(results);
});
Output:
async4
async2
async1
async3
Results:
[ { err: null, resp: { name: 'async1' } },
{ err: null, resp: { name: 'async2' } },
{ err: null, resp: { name: 'async3' } },
{ err: null, resp: { name: 'async4' } } ]

Related

In Express.js how can we make res.send to wait for data to be fetched from database

The method res.send() returns (empty) before the data is fetched from database (Mongodb) from the method
Skills.find({ skillbranch: branches[i]._id }, function (err, skills) {
How can we add await or async to this before returning?
I'm learning node/express and was not able to make up the syntax to use with some answers/examples on other posts on stackoverflow where they were using async , await, Promise.
const Skills = require('../models/skills.model.js');
const SkillBranch = require('../models/skillbranch.model.js');
exports.getSkills = function (req, res) {
let branchSkills = [];
SkillBranch.find(function (err, branches) {
if (branches) {
var obj = {
"status": "200",
"message": "skills",
"data": branches
}
for (var i = 0; i < branches.length; i++) {
Skills.find({ skillbranch: branches[i]._id }, function (err, skills) {
console.log(JSON.stringify(skills)); //this is returning after res.send()
branchSkills.push(skills);
})
if (i == branches.length - 1) {
var obj = {
"status": "200",
"message": "skills",
"data": branchSkills
}
//this is returning before Skills.find() is complete
res.send(JSON.stringify(obj));
}
}
} else {
var obj = {
"status": "500",
"message": "Getting skills ",
"data": []
}
res.send(JSON.stringify(obj));
}
})
};
Something like this should work.
Create a promise on the mongo call. Once it completes, just convert it to an array, push it to your branch skills, and resolve the promise with your new updated version. Then send data with that new updated version rather than your global branchSkills object you've been using.
Should be noted i'm not sure what you're using branch skills for, but you may just be able to resolve with skills without pushing to that global array? This should fix your timing issues though.
new Promise((resolve, reject) => {
Skills.find({ skillbranch: branches[i]._id }).toArray((err, skills) => {
if (err) {
reject(err);
} else {
branchSkills.push(skills);
resolve(branchSkills);
}
});
});
}).then(updatedBranchSkills => {
if (i == branches.length - 1) {
var obj = {
"status": "200",
"message": "skills",
"data": updatedBranchSkills
}
res.send(JSON.stringify(obj));
});
You must call res.send() inside the Skills.find() callback, not outside it. But since you are calling multiple Skills.find() you need to keep track of how many callbacks have returned. For this you can use a variable:
// Keep track of completed find():
var completed = 0;
for (var i = 0; i < branches.length; i++) {
Skills.find({ skillbranch: branches[i]._id }, function (err, skills) {
console.log(JSON.stringify(skills));
completed++;
branchSkills.push(skills);
if (completed == branches.length) {
var obj = {
"status": "200",
"message": "skills",
"data": branchSkills
}
res.send(JSON.stringify(obj)); // INSIDE callback
}
}) // <--- note that callback ends here!
}

Nodejs crashes when database connection fails

While my database server is not available and any function of my node-express rest service like hiExpress is called, Nodejs crashes the node server and console of node reports
sql server connection closed
I do not want this to happen because either it should go to err function or at least it must be cautht by catch block. What could i do to avoid the crash of nodejs when database server is not available I am using following code which is absolutely fine as long as database server is available.
var sqlServer = require('seriate');
app.get('/hiExpress',function(req, res)
{
var sr = {error:'',message:''};
var sql= 'select * from table1 where id=? and name=?';
var params = {id: 5, name:'sami'};
exeDB(res,sr,sql, params);//sent only 4 parameters (not 6)
});
function exeDB(res, sr, sql, params, callback, multiple) {
try {
var obj = {};
for (p in params) {
if (params.hasOwnProperty(p)) {
obj[p] = {
type: sqlServer.VARCHAR,
val: params[p]
};
}
};
var exeOptions = {
query: sql,
params: obj
};
if (multiple) {
exeOptions.multiple = true;
}
sqlServer.execute(sqlServerConfigObject, exeOptions).then(function (results) {
sr.data = results;
if (callback)
callback(sr);
else
res.json(sr); //produces result when success
}, function (err) {
//sr.message = sql;
console.log(11);
sr.error = err.message;
res.json(sr);
});
}
catch (ex) {
console.log(21);
sr.error = ex.message;
res.json(sr);
}
}
Why I preferred to use seriate
I had not been much comfortable with node-SQL, especially when when it came to
multiple queries option even not using a transaction. It facilitates easy go to parameterized queries.
You can use transaction without seriate but with async like below
async.series([
function(callback) {db.run('begin transaction', callback)},
function(callback) {db.run( ..., callback)},
function(callback) {db.run( ..., callback)},
function(callback) {db.run( ..., callback)},
function(callback) {db.run('commit transaction', callback)},
], function(err, results){
if (err) {
db.run('rollback transaction');
return console.log(err);
}
// if some queries return rows then results[query-no] contains them
})
The code is very dirty. Pass req and res params to db-layer is not a good idea.
Try change exeDB. I'm not sure, but probably you don't set error catcher to promise
function exeDB(res, sr, sql, params, callback, multiple) {
// It will execute with no error, no doubt
var obj = {};
for (p in params) {
if (params.hasOwnProperty(p)) {
obj[p] = {
type: sqlServer.VARCHAR,
val: params[p]
};
}
};
var exeOptions = {
query: sql,
params: obj
};
if (multiple) {
exeOptions.multiple = true;
}
// Potential problem is here.
// Catch is useless because code below is asynchronous.
sqlServer.execute(sqlServerConfigObject, exeOptions).then(function (results) {
sr.data = results;
if (callback)
callback(sr);
else
res.json(sr); //produces result when success
}).error(function(err){ // !!! You must provide on-error
console.log(err);
};
}

Validating all data with DB before bulk insert mongoose/mongodb

I am trying to validate the array of objects before inserting them into the mongodb.
what i am trying to do is, lets say i have an object like below
var data= { prodDetails:
[
{ measured: 'Liters',
name: 'A',
prodCode: '713',
status: true },
{ measured: 'Liters',
name: 'B',
prodCode: '713',
status: true },
{ measured: 'Liters',
name: 'C',
prodCode: '674',
status: true }
]
};
before making a bulk insert call i want check whether the given prodCode is valid DB Code or not and name duplicated or not
i am using node bluebird promises.
i tried the following code to validate prodCode
var bulkOperations = {
bulkProdInsert: function (body) {
return new Promise(function (reslv, rej) {
if (body.prodDetails.length > 0) {
common_lg.getValueById(body, "typesProd", body.prodDetails[0].prodCode).then(bulkOperations.successCallback(reslv, rej, body)).catch(bulkOperations.errCallback(reslv, rej, body));
};
reslv();
});
},
successCallback: function (reslv, rej, body) {
return function (res) {
if (res) {
body.prodDetails.splice(0, 1);
if (body.prodDetails.length > 0) {
common_lg.getValueById(body, "typesProd", body.prodDetails[0].prodCode).then(bulkOperations.successCallback(reslv, rej, body)).catch(bulkOperations.errCallback(reslv, rej, body));
}
};
};
},
errCallback: function (reslv, rej, body) {
return function (err) {
body.prodDetails.splice(0, 1);
if (body.prodDetails.length > 0) {
common_lg.getValueById(body, "typesProd", body.prodDetails[0].prodCode).then(bulkOperations.successCallback(reslv, rej, body)).catch(bulkOperations.errCallback(reslv, rej, body));
};
};
}
};
but i want to do is insert all the objects/documents into DB when name and prodCode of all the objects/documents is validated.
how to achieve this.
thanks
It sounds like you want to check the input object and then make DB calls. I would suggest the .map method of bluebird.
var promise = require('bluebird');
var checkValues = promise.method( function(prod){
if( isValid(prod.prodCode) ){
return prod;
}
//something went wrong!
throw new Error('prodCode ' + prod.prodCode + ' is invalid');
}
promise.map( data.prodDetails, checkValues )
.then(function(){
//it worked! You can call the DB now.
})
.catch(function(error){
//something went wrong, look at that error (or pass along)
})
So long as your checkValues method is a promise, you can run it against every value in your input array, and use the success of .then to know things worked and call your DB!

Async function in nodejs loop

I'm having problem calling async function inside a while loop.
the problem is 'while' statement will end before its underlying function result appear and thats because it's async function.
the code is like below:
while (end < min) {
db.collection('products').count({
tags: {
$in: ['tech']
}
}, function(err, result) {
if (result) {
a = result;
}
});
max = min;
min = max - step;
myitems.push(a);
}
res.send(myitems);
and at the end i could not send the result because all of while iteration should finish before sending the final result.
how could i modify the code to solve such a problem?
thanks in advance
Without using third party libraries, here's a method of manually sequencing your async operations. Note, because this is async, you have to process the results inside of the next() function when you see that you are done iterating.
// assume that end, max, min and step are all defined and initialized before this
var results = [];
function next() {
if (end < min) {
// something seems missing from the code here because
// this db.collection() call is always the same
db.collection('products').count({tags: {$in: ['tech']}}, function(err, result) {
if (!err && result) {
results.push(result);
max = min;
min - max - step;
next();
} else {
// got an error or a missing result here, provide error response
console.log("db.collection() error or missing result");
}
}
} else {
// all operations are done now
// process the results array
res.send(results);
}
}
// launch the first iteration
next();
You could leverage a 3rd party library to do this as well (non working performQuery example using async):
function performQuery(range, callback) {
// the caller could pre calculate
// the range of products to retrieve
db.collection('products').count({
tags: {
$in: ['tech'],
// could have some sort of range query
$gte: range.min,
$lt: range.max
}
}, function(err, result) {
if (result) {
callback(result)
}
});
}
async.parallel([
performQuery.bind(null, {min: 0, max: 10}),
performQuery.bind(null, {min: 10, max: 20})
], function(err, results) {
res.send(results);
});

Create/update with Sequelize on an array of items

I created a function to:
take an array of 'labels' and look for whether they have a record in the db already
create those which don't exist,
and update those which do exist
return a json array reporting on each item, whether they were updated/created, or resulted in an error
I managed to make it work but I feel like I just made some ugly dogs' dinner!
var models = require("../models");
var Promise = models.Sequelize.Promise;
module.exports = {
addBeans: function (req, callback) {
Promise.map(req.body.beansArr, function (bean) {
return models.Portfolio.findOrCreate({where: {label: bean}}, {label: bean});
}).then(function (results) { // Array of 'instance' and 'created' for each bean "findOrCreate(where, [defaults], [options]) -> Promise<Instance>"
var promisesArr = [];
results.forEach(function (result) {
if (result[1]) { // result[1] = wasCreated
promisesArr.push(Promise.resolve([result[0].dataValues.label, "created"]));
} else {
promisesArr.push(
models.Portfolio.update({label: result[0].dataValues.label},
{where: {label: result[0].dataValues.label}}).then(function () {
return Promise.resolve([result[0].dataValues.label, "updated"])
})
);
}
});
return promisesArr;
// When it's all done create a JSON response
}).then(function (results) {
var resultObj = {items: []}; // JSON to return at the end
Promise.settle(results).then(function (promiseinstances) {
for (var i = 0; i < promiseInstances.length; i++) {
if (promiseInstances[i].isFulfilled()) {
resultObj.items.push({
item: {
label: promiseInstances[i].value()[0],
result: promiseInstances[i].value()[1],
error: ''
}
});
}
else if (promiseInstances[i].isRejected()){
resultObj.items.push({
label: promiseInstances[i].value()[0],
result: 'error',
error: promiseInstances[i].reason()
});
}
}
// Send the response back to caller
}).then(function () {
return callback(null, resultObj);
}, function (e) {
return callback(e, resultObj);
});
});
}
};
Question:
Is there an easier or more obvious way to create/update values with Sequelize?
Is my use of Promise.settle() appropriate for this case? I have the feeling I made this more complicated than it needs to be.
I am new to Sequelize and using Promises, I'd appreciate if someone could advise on this.
I feel like this would work better on CodeReview.SE but I can see a few issues.
Is there an easier or more obvious way to create/update values with Sequelize?
Well, for one thing:
.then(function(array){
var newArr = [];
array.forEach(function(elem){
newArr.push(fn(elem);
}
return newArr;
});
Is just
.map(fn)
Additionally, promises assimilate so you can return val; from a .then you don't have to return Promise.resolve(val);.
So:
).then(function (results) { // Array of 'instance' and 'created' for each bean "findOrCreate(where, [defaults], [options]) -> Promise<Instance>"
var promisesArr = [];
results.forEach(function (result) {
if (result[1]) { // result[1] = wasCreated
promisesArr.push(Promise.resolve([result[0].dataValues.label, "created"]));
} else {
promisesArr.push(
models.Portfolio.update({label: result[0].dataValues.label},
{where: {label: result[0].dataValues.label}}).then(function () {
return Promise.resolve([result[0].dataValues.label, "updated"])
})
);
}
});
return promisesArr;
})
Is just
.map(function(result){
if(result[1]) return [result[0].dataValues.label, "created"];
return models.Portfolio.update({label: result[0].dataValues.label},
{where: {label: result[0].dataValues.label}}).
return([result[0].dataValues.label, "updated"]);
});
However, since you want it to work regardless of it being resolved, you'd have to do:
.then(function(results){
return results.map(function(result){
if(result[1]) return [result[0].dataValues.label, "created"];
return models.Portfolio.update({label: result[0].dataValues.label},
{where: {label: result[0].dataValues.label}}).
return([result[0].dataValues.label, "updated"]);
});
});
Which means it'll resolve regardless, then you'd call .settle():
.settle().then(function(results){
// your settle logic here
});
Note that the last:
}).then(function () {
return callback(null, resultObj);
}, function (e) {
return callback(e, resultObj);
});
Is simply:
.nodeify(callback);
However, I recommend sticking to promises.
I use Promise.settle for sequelize.update, and can get affect rows number by _settledValueField .
promise.push(...update...)
db.sequelize.Promise.settle(promise).then(function (allresult) {
var affectcnt = 0
allresult.forEach(function (singlecnt) {
if (undefined !== singlecnt._settledValueField[1]) {
affectcnt += parseInt(singlecnt._settledValueField[1])
}
})
unfortunately, it's only work for update.
You can insert array in database using sequelize. You want to change in model like below. I am trying to add multiple languages in database through array.
language: { type: DataTypes.STRING,
allowNull: false,
get()
{
return this.getDataValue('language').split(';')
},
set(val)
{
this.setDataValue('language',Array.isArray(val) ? val.join(','):val);
}
}

Resources