mongoose queries execution order - node.js

I'm having a problem in My code using nodejs and mongoose. where I have a function that is supposed to get books from mongodb ( book.findOne) then update them using second query (updateMany).
the problem is that UpdateMany query executes before getting the books ..so my array stays empty and nothing is updated.
I know that node.js is asynchronous but how could i solve this ?
function UpdateBulk(completeValidBooks){
var existingRowsInDb = [];
completeValidBooks.forEach(function(currentBook) {
book.findOne({'ISIN': currentBook.ISIN },
function(error, result) {
existingRowsInDb.push(result);
});
});
book.updateMany(existingRowsInDb, (err,docs) => {
err ? console.log(err) : console.log(`updated ${docs.length}`);
});
}

Using promise you can do something like this:
function UpdateBulk(completeValidBooks) {
GetExistingRows(completeValidBooks).then((existingRowsInDb) => {
book.updateMany(existingRowsInDb, (err, docs) => {
err ? console.log(err) : console.log(`updated ${docs.length}`);
});
})
}
function GetExistingRows(completeValidBooks) {
return new Promise((resolve, reject) => {
var existingRowsInDb = [];
completeValidBooks.forEach((currentBook) => {
book.findOne({
'ISIN': currentBook.ISIN
}, (error, result) => {
existingRowsInDb.push(result);
});
});
resolve(existingRowsInDb);
})
}
Using callback you can do like this:
function UpdateBulk(completeValidBooks) {
GetExistingRows(completeValidBooks, (existingRowsInDb) => {
book.updateMany(existingRowsInDb, (err, docs) => {
err ? console.log(err) : console.log(`updated ${docs.length}`);
});
})
}
function GetExistingRows(completeValidBooks, callback) {
var existingRowsInDb = [];
completeValidBooks.forEach((currentBook) => {
book.findOne({
'ISIN': currentBook.ISIN
}, (error, result) => {
existingRowsInDb.push(result);
});
});
callback(existingRowsInDb);
}

Related

util.promisify old library call

I've been scratching my head over on how to promisify this call and return data to another function when it becomes available.
syno.query('/webapi/query.cgi', {
api : 'SYNO.API.Info',
version: 1,
method : 'query',
query : 'ALL'
}, function(err, data) {
if (err) return console.error(err);
console.log(data);
});
Can anybody shed some light here ? I'm new to nodejs
the old library that I'm using is this one https://www.npmjs.com/package/synology
try :
const query = () => new Promise((resolve, reject) => {
syno.query('/webapi/query.cgi', {
api: 'SYNO.API.Info',
version: 1,
method: 'query',
query: 'ALL'
}, function(err, data) {
if (err) reject(err);
resolve(data);
});
})
then you can call :
query().then(data => { //do something
})
.catch(e => { //do other thing
})

Saving replaced string data in mongodb using nodejs

I am trying to replace a string in url . Here is image of it
in this image I want to replace lssplalpha with lssplprod which are in pics array. For that I created an api . Here is a code
apiRoutes.get('/SchoolListing_lssplalpha_Replace',function(req, res) { schoolListModel.find({},function(err,check){
if(err){
return console.log(err);
}
else{
for(var i=0;i<check.length;){
var f=0;
for(var j=0;j<check[i].pics.length;j++){
f++;
var newTitle = check[i].pics[j].replace("lssplalpha","lsslprod");
check[i].pics[j] = newTitle;
console.log("after change",check[i].pics[j]);
check[i].save(function (err) {
if(err) {
console.error('ERROR!');
}
});
}
if(j==check[i].pics.length&&j==f){
i++;
}
}
console.log("i value",i);
console.log("check length",check.length);
if(i==check.length){
return res.json({status:true,message:"Updated Schools"}); }
}
});
});
I am getting success response . When I go and check database nothing changed in db. To know the reason I write log of it. When I see logs it was replacing correctly. But I didn't understand why those are not reflecting in database? Here is an image of log in console
Please help me to come out of this
The issue here is you are running a for loop (synchronous) where you are calling the model.save() operation which is asynchronous and the loop keeps iterating but the results of the async calls come later. The process of saving a database item in an array takes some time and Node.js knows this, so it starts the update and then just moves on trying to update the next item in the array. Once the write operation is complete a callback function is run, but by that point the loop has completed and there is no way to know which items finish in what order.
You could use the Bulk Write API to update your models. This allows you to sends multiple write operations to the MongoDB server in one command. This is faster than sending multiple independent operations (like) if you use create()) because with bulkWrite() there is only one round trip to MongoDB.
The following examples show how you can use the bulkWrite.
Using async/await:
apiRoutes.get('/SchoolListing_lssplalpha_Replace', async (req, res) => {
try {
let ops = [];
const docs = await schoolListModel.find({}).lean().exec();
docs.forEach(doc => {
const pics = doc.pics.map(pic => pic.replace("lssplalpha", "lsslprod"));
ops.push({
"updateOne": {
"filter": { "_id": doc._id },
"update": {
"$set": { pics }
}
}
});
});
const result = await schoolListModel.bulkWrite(ops);
console.log('Bulk update complete.', result);
res.status(200).json({
status: true,
message: "Updated Schools"
});
} catch (err) {
res.status(400).send({
status: false,
message: err
});
}
});
Using Promise API:
const bulkUpdate = (Model, query) => (
new Promise((resolve, reject) => {
let ops = [];
Model.find(query).lean().exec((err, docs) => {
if (err) return reject(err);
docs.forEach(doc => {
const pics = doc.pics.map(pic => (
pic.replace("lssplalpha", "lsslprod")
));
ops.push({
"updateOne": {
"filter": { "_id": doc._id },
"update": {
"$set": { pics }
}
}
});
if (ops.length === 500) {
Model.bulkWrite(ops).then((err, result) => {
if (err) return reject(err);
ops = [];
resolve(result);
});
}
});
if (ops.length > 0) {
Model.bulkWrite(ops).then((err, result) => {
if (err) return reject(err);
resolve(result);
});
}
});
})
);
apiRoutes.get('/SchoolListing_lssplalpha_Replace', (req, res) => {
bulkUpdate(schoolListModel, {}).then(result => {
console.log('Bulk update complete.', result);
res.status(200).json({
status: true,
message: "Updated Schools"
});
}).catch(err => {
res.status(400).send({
status: false,
message: err
});
});
});
You are running asynchronous call model.save() in for loop(synchronous)
To make your for loop synchronous you can use for...of loop which works asynchronous, also you will not need to add multiple checks like you have done in your code.
Try following code, it will work
apiRoutes.get('/SchoolListing_lssplalpha_Replace', function (req, res) {
schoolListModel.find({},function (err, check) {
if (err) {
return console.log(err);
}
else {
for (let checkObj of check) {
let newPicArr=[];
for (let pic of checkObj.pics) {
pic = pic.replace("lssplalpha", "lsslprod");
newPicArr.push(pic);
}
checkObj.pics=newPicArr;
checkObj.save(function (err) {
if (err) {
console.error('ERROR!');
}
});
}
return res.json({ status: true, message: "Updated Schools" });
}
});
});

Node js foreach done function

Insted of using setTimeout, what should I use after foreach complete?
app.post('/grid', function(req, res){
getResults(req.body.idarray, function(callback){
res.send(callback);
});
});
function getResults(userIds, callback) {
var totalresult = [];
userIds.forEach(function (user) {
sequence
.then(function (next) {
db.query('SELECT given FROM books WHERE user_id = ?', [user.userId], function (err2, result) {
if (err2) throw err2;
next(err, result);
});
})
.then(function (next, err, books) {
db.query('SELECT received FROM encycs WHERE user_id = ?', [user.userId], function (err3, result2) {
if (err3) throw err3;
next(err, result2, books);
});
})
.then(function (next, err, books, encycs ) {
Calculation(books, encycs, function (cb) {
totalresult.push(cb);
});
next();
});
});
setTimeout(function() {
console.log(totalresult); // output ok.
return callback(totalresult); // returning as expected
}, 2000);
}
I dont know what totalresult.length is. So i can't check the length.
So, according to your use case you need to call callback somehow and pass totalresult into it, because that what your external code, code in the route expected.
To do that, you can call callback before calling next of the third .then statement. Like that.
...
.then(function (next, err, books, encycs ) {
Calculation(books, encycs, function (cb) {
totalresult.push(cb);
});
callback(totalresult);
next();
//console.log(totalresult); //output OK.
});
This might work.
Update 1
It is hard to follow with your code. Can't catch up the logic of it. I would propose you Promises approach. I prepared that solution, that might work. It might contain little errors, but it represents the main idea of what you are trying to achieve, and how it can be done.
app.post("/grid", (req, res) => {
getResults(req.body.idarray)
.then(data => {
res.status(200).json(data);
})
.catch(err => {
console.error("Error occured", err);
res.status(500).json(err);
});
});
function getResults(userIds) {
let promises = userIds.map(loadCalculation);
//this will wait until all loadings are done
return Promise.all(promises);
}
function loadCalculation(user) {
//parallel loading of the books and encycs
return Promise.all([loadBooks(user), loadEncycs(user)])
.then(results => {
let books = results[0];
let encycs = results[1];
let totalresult = [];
Calculation(books, encycs, function (cb) {
totalresult.push(cb);
});
return totalresult;
});
}
function loadBooks(user) {
return makeQuery('SELECT given FROM books WHERE user_id = ?', user);
}
function loadEncycs(user) {
return makeQuery('SELECT received FROM encycs WHERE user_id = ?', user);
}
function makeQuery(query, user) {
return Promise((resolve, reject) => {
db.query(query, [user.userId], function (err, result) {
if(err) {
reject(err);
} else {
resolve(result);
}
});
});
}
Please, note that this is not really performant way to load the data from database, at least, I'm sure that you there is a possibility to load all the books and encycs with a single query, because you are using SQL, and it is really flexible language.

Resolve not working in loop Node.js

Hi I have a problem running a loop and getting the return data using Promises.
I have a getStudentMarks method for getting students marks from the database in subject wise.
getStudentMarks: function(studentId, studentStandard) {
console.log("getStudentMarks invoked...");
return new Promise(function(resolve, reject) {
r.table('student_subjects').filter({
"studentId": studentId,
"studentStandard": studentStandard
}).pluck("subjectId", "subjectName").run(connection, function(err, cursor) {
if (err) {
throw err;
reject(err);
} else {
cursor.toArray(function(err, result) {
if (err) {
throw err
} else {
console.log(result.length);
if (result.length > 0) {
studentSubjectArray = result;
var studentMarksSubjectWiseArray = [];
studentSubjectArray.forEach(function(elementPhoto) {
r.table('student_marks').filter({
"studentId": studentId,
"subjectId": studentSubjectArray.subjectId
}).run(connection, function(err, cursor) {
if (err) {
throw err;
reject(err);
} else {
cursor.toArray(function(err, result_marks) {
var studnetMarksDataObject = {
subjectId: studentSubjectArray.subjectId,
subjectName: studentSubjectArray.subjectName,
marks: result.marks
};
studentMarksSubjectWiseArray.push(studnetMarksDataObject);
});
}
});
});
resolve(studentMarksSubjectWiseArray);
}
}
});
}
});
});
}
I'm invoking the method by,
app.post('/getStudentMarks', function(req, reqs) {
ubm.getStudentMarks(req.body.studentId, req.body.studentStandard)
.then((data) => {
console.log('return data: ' + data);
})
.catch((err) => {
console.log(err);
});
});
When I run the code its working absolutely fine there is no error. I get all the student marks object in the studentMarksSubjectWiseArray array. But the problem is even before the studentSubjectArray loops gets completed, the resolve is getting executed and I'm getting a blank array as return. How do I solve the problem. I understand that I'm not doing the Promises right. I'm new to Promises so I'm not being able to figure out the right way.
That happens because inside your studentSubjectArray.forEach statement you perform set of asynchronous operations r.table(...).filter(...).run() and you push their result into the array. However, those actions finish after you perform the resolve(), so the studentMarksSubjectWiseArray is still empty. In this case you would have to use Promise.all() method.
let promisesArray = [];
studentSubjectArray.forEach((elementPhoto) => {
let singlePromise = new Promise((resolve, reject) => {
// here perform asynchronous operation and do the resolve with single result like r.table(...).filter(...).run()
// in the end you would perform resolve(studentMarksDataObject)
r.table('student_marks').filter({
"studentId": studentId,
"subjectId": studentSubjectArray.subjectId
}).run(connection, function(err, cursor) {
if (err) {
throw err;
reject(err);
} else {
cursor.toArray(function(err, result_marks) {
var studnetMarksDataObject = {
subjectId: studentSubjectArray.subjectId,
subjectName: studentSubjectArray.subjectName,
marks: result.marks
};
resolve(studnetMarksDataObject);
});
}
});
});
promisesArray.push(singlePromise)
});
Promise.all(promisesArray).then((result) => {
// here the result would be an array of results from previously performed set of asynchronous operations
});

Node async callback was already called when trying to make a nested query

I am getting a Callback was already called error while trying to make asynchronous queries using the MEAN stack. I need the second callback to only trigger after the nested query has been completed (as per the comments in the code). Why am I getting this error?
Example of the route:
router.route('/teams/:user_id').get(function (req, res) {
TeamProfile.find({
Members : {
$in : [req.params.user_id]
}
}).exec(function (err, teamProfiles) {
var asyncTasks = [];
teamProfiles.forEach(function (teamProfile) {
asyncTasks.push(function (callback) {
UserProfile.find({
UserID : {
$in : teamProfile.Members.map(function (id) {
return id;
})
}
}, function (err, userProfiles) {
teamProfile.Members = userProfiles;
callback();
})
});
});
teamProfiles.forEach(function (teamProfile) {
asyncTasks.push(function (callback) {
Draft.find({
_id : {
$in : teamProfile.Drafts.map(function (id) {
return id;
})
}
}, function (err, drafts) {
teamProfile.Drafts = drafts;
drafts.forEach(function (draft) {
Comment.find({
_id : {
$in : draft.Comments.map(function (id) {
return id;
})
}
}).exec(function (err, comments) {
draft.Comments = comments;
callback();
//This is where the callback should be called
//Throws Error: Callback was already called.
})
})
})
});
});
async.parallel(asyncTasks, function () {
res.json(teamProfiles)
});
});
})
I am using async.parallel() to perform all the queries. I am very new to all of this so please excuse some beginner mistakes.
You are iterating over drafts synchronously and calling async's callback function on the first item. Getting an error when you try to call it again with the second item is expected behaviour.
You should instead call the done callback once you have finished all the draft queries, not for each one. Since you are using async, you could nest another async.each to handle this:
router.route('/teams/:user_id').get(function (req, res) {
TeamProfile.find({
Members : {
$in : [req.params.user_id]
}
}).exec(function (err, teamProfiles) {
var asyncTasks = [];
teamProfiles.forEach(function (teamProfile) {
asyncTasks.push(function (callback) {
UserProfile.find({
UserID : {
$in : teamProfile.Members.map(function (id) {
return id;
})
}
}, function (err, userProfiles) {
teamProfile.Members = userProfiles;
callback();
});
});
});
teamProfiles.forEach(function (teamProfile) {
asyncTasks.push(function (callback) {
Draft.find({
_id : {
$in : teamProfile.Drafts.map(function (id) {
return id;
})
}
}, function (err, drafts) {
teamProfile.Drafts = drafts;
async.each(drafts, function(draft, draftCallback){
Comment.find({
_id : {
$in : draft.Comments.map(function (id) {
return id;
})
}
}).exec(function (err, comments) {
draft.Comments = comments;
draftCallback();
});
}, function(err){
callback();
});
});
});
});
async.parallel(asyncTasks, function () {
res.json(teamProfiles)
});
});
});

Resources