Mongodb, Node - Updating multiple objects synchronously - node.js

Using Mongodb, Nodejs, Async.js and Express.js
I am trying to update multiple documents, where each document has its own update, at the same time. I want to wait for all documents to update so that I can notify user that all documents have updated.
The issue I am having now is that my callback function is not firing, or if it is, then nothing is happening. Here is my progress:
db.client.collection('page').find({page_id: page_id}).toArray(function(page_err, document_page) {
if(page_err) {
throw page_err;
} else if(document_page === '' || document_page === undefined || document_page === null) {
throw page_err;
} else {
var count = 0;
async.each(data, function iteratee(i, callback) {
var item_id = (i.item_id === '') ? new ObjectId() : new ObjectId(i.item_id);
var query = {item_id: item_id};
var update = {
_id : new ObjectId(),
page_id : page_id,
section_id : null,
item_id : item_id,
created : new Date().toISOString(),
item_type : "dish",
item: {
title: i.title,
description: i.description,
price: i.price,
star: false,
double_star: false
},
last_modified: new Date().toISOString()
};
var options = { upsert: true };
db.client.collection('item').updateOne(query, {$set: update}, options, function(item_err, results) {
if(item_err) {
res.sendStatus(500);
} else if(results === '' || results === undefined || results === null) {
res.sendStatus(400);
} else {
++count;
if(count === data.length) {
callback();
return;
}
}
});
}, function() {
console.log('sending 200 status');
res.sendStatus(200);
});
}
});
When I run the code I do enter the if statement where I call callback(). I have been stuck on this for a few hours and I cannot get it to work. If you need more info, I'd be happy to provide it. For simplicity's sake I removed many console.logs to avoid clutter as well.

All iterations need to fire the callback otherwise it will hang indefinitely. callback must be called in every iteration. Always.
If you encounter an error, you need to call callback(error). The problem you'll have is that async.each schedules all iterations beforehand so iteratee will fire data.length times regardless of whether an error is encountered half way through execution or not. If you need to run them in series you can use async.eachSeries which will take more time but gives you better control and no need to rollback.
So code wise it looks like this:
db.client.collection('page').find({page_id: page_id}).toArray(function(page_err, document_page) {
if(page_err) {
throw page_err;
} else if(document_page === '' || document_page === undefined || document_page === null) {
throw page_err;
} else {
async.each(data, function iteratee(i, callback) {
var item_id = (i.item_id === '') ? new ObjectId() : new ObjectId(i.item_id);
var query = {item_id: item_id};
var update = {
_id : new ObjectId(),
page_id : page_id,
section_id : null,
item_id : item_id,
created : new Date().toISOString(),
item_type : "dish",
item: {
title: i.title,
description: i.description,
price: i.price,
star: false,
double_star: false
},
last_modified: new Date().toISOString()
};
var options = { upsert: true };
db.client.collection('item').updateOne(query, {$set: update}, options, function(item_err, results) {
if(item_err) {
callback(500);
} else if(results === '' || results === undefined || results === null) {
callback(400)
} else {
callback();
}
});
}, function(err) {
// Passing the status code only for the example.
// `err` should be an object with more metadata probably
if(err) {
res.sendStatus(err);
return;
}
console.log('sending 200 status');
res.sendStatus(200);
});
}
});

Related

Problem with Mongoose pre.save() generating specific codes on record creation

I have a Mongo collection with this properties:
{
"_id" : ObjectId("384f1f06f72cc1b566e32f98"),
"num" : 41,
"product" : ObjectId("5c8921d8f9f7be241c0b02cf"),
"data" : {
"phone" : "123123123",
"email" : "email#email.com",
"name" : "John",
"_id" : ObjectId("5ca34689ac024b579991fe26")
},
"generatedCode": "01-1FCS3";
}
Depending on the type of product (it's an object as you can see), I want to generate a different code every time a new element is created
ProductSchema.pre('save', function(next) {
if ((typeof this.generatedCode === 'undefined') || !this.generatedCode) {
ProductSchema.find({_id: this.product}, function(error, existingProduct) {
if (error) {
next(Error(error));
}
else {
if (existingProduct.length > 0) {
console.log("Type of product: "+ existingProduct[0].type);
generateCode(this,existingProduct[0].type).then(function(doc) {
console.log('Code generated:' + doc.generatedCode);
next();
}, function(err) {
console.error(err);
next(err);
});
}
else {
next(new Error("Error: Not found."));
}
}
});
} else {
next();
}
});
function generatedCode(doc,productType){
var deferred = q.defer();
try {
console.log("Product type = "+ invoiceType);
var ObjectId = require('mongoose').Types.ObjectId;
var generatedCode = '';
switch(productType) {
case 'TYPE1':
generatedCode = 'COD01-'+ random(5);
break;
case 'TYPE2':
generatedCode = 'COD02-'+ random(5);
break;
default: // other types
generatedCode = 'COD03-'+ random(5);
}
console.log("generatedCode = "+ generatedCode);
var InvoiceModel = mongoose.model('Invoice', InvoiceSchema);
// check that there are no records with that code
InvoiceModel.find({generatedCode: generatedCode}, function(err, existingInvoice) {
console.log('Find: '+ generatedCode);
if(err) {
console.log(err);
deferred.reject(err);
}
if(existingInvoice.length > 0) {
console.log('Alredy exists ' + generatedCode + ' in db');
generatedCode(doc,productType).then(function(doc) {
deferred.resolve(doc);
}, function(err) {
deferred.reject(err);
});
}
else {
console.log("generatedCode = "+ generatedCode);
console.log("Doc = "+ doc);
doc.generatedCode = generatedCode;
deferred.resolve(doc);
}
});
}
catch(exception) {
console.error(exception);
deferred.reject(new Error(exception.message));
}
return deferred.promise;
}
I do not know why the error is due, but in the line "doc.generatedCode = generatedCode;" returns null It seems as if doc does not exist and I do not understand what is failing...
I show you the debug:
Type of product: GENERIC
Product type = GENERIC
generatedCode = COD03-8KORD
Find: COD03-8KORD
generatedCode = COD03-8KORD
Doc = null
[ERROR] (node.js:496) -> uncaughtException: Cannot set property 'code' of null
The value of this changes when you are inside a callback function. Same is happening to your code here. When you do ProductSchema.find and try to access this inside it, the value is changed. You need to save a reference to actual value outside.
ProductSchema.pre('save', function(next) {
if ((typeof this.generatedCode === 'undefined') || !this.generatedCode) {
var that = this; // here create a reference to "this"
ProductSchema.find({_id: this.product}, function(error, existingProduct) {
..
..
// generateCode(this,existingProduct[0].type).then(function(doc) { // YOUR OLD CODE
generateCode(that,existingProduct[0].type).then(function(doc) { // here, change "this" to "that" which is the actual reference of your object
..
..

The ultimate way to prevent duplication in Parse Server once and for all

One of the biggest issue we face now with parse-server is duplication. Although we have implemented a Parse cloud code to prevent such event through beforeSave and afterSave methods at the same time added external middleware to check for existing object before saving still we face duplication over and over specially on concurrent operations.
Here is our code to prevent duplication for a specific class:
Parse.Cloud.beforeSave("Category", function(request, response) {
var newCategory = request.object;
var name = newCategory.get("name");
var query = new Parse.Query("Category");
query.equalTo("name", name);
query.first({
success: function(results) {
if(results) {
if (!request.object.isNew()) { // allow updates
response.success();
} else {
response.error({errorCode:400,errorMsg:"Category already exist"});
}
} else {
response.success();
}
},
error: function(error) {
response.success();
}
});
});
Parse.Cloud.afterSave("Category", function(request) {
var query = new Parse.Query("Category");
query.equalTo("name", request.object.get("name"));
query.ascending("createdAt");
query.find({
success:function(results) {
if (results && results.length > 1) {
for(var i = (results.length - 1); i > 0 ; i--) {
results[i].destroy();
}
}
else {
// No duplicates
}
},
error:function(error) {
}
});
});
This code above is able to prevent some duplicate but most still goes through, example:
What is the "ultimate way" to prevent duplication with Parse server?
You can always create a unique index in mongodb for the field that should be unique in your document.
This way any save that conflicts with that index, will be aborted
Maybe you should write something with Promises like :
Parse.Cloud.beforeSave("Category", function (request, response) {
return new Promise((resolve, reject) => {
var query = new Parse.Query("Category");
query.equalTo("name", "Dummy");
return query.first().then(function (results) {
resolve(); // or reject()
});
})
});
Parse.Cloud.beforeSave("Category", async (request) => {
(...)
await results = query.first();
// then your logic here
response.success();
response.error({ errorCode: 400, errorMsg: "Category already exist" })
})
Here is my Solution:
Parse.Cloud.beforeSave( 'ClassName', async ( request ) => {
const columnName = 'columnName'
const className = 'ClassName'
if( request.object.isNew() ) {
var newCategory = request.object
var name = newCategory.get( columnName )
var query = new Parse.Query( className )
query.equalTo( columnName, name )
const results = await query.count()
if( results === 0 ) {
// no response.success needed
// https://github.com/parse-community/parse-server/blob/alpha/3.0.0.md
} else {
throw 'Is not unique';
}
}
} )

returning Mongoose query result from Async call

I'm working on a problem where I need to query the db for an instance of a Voter, and use that instance to update an Election, returning to the original function whether that update was successful or not. My code currently looks like this:
function addCandidatesToElection(req, res) {
let electionName = req.body.electionName;
let candidates = req.body.candidates;
let addedCandidatesSucessfully = true;
for(let i=0; i<candidates.length; i++) {
addedCandidatesSucessfully = _addCandidateToElection(electionName, candidates[i]);
console.log("added candidates sucessfully:" + addedCandidatesSucessfully);
}
if(addedCandidatesSucessfully) {
res.send("createElection success");
} else {
res.send("createElection fail");
}
}
which calls this function:
function _addCandidateToElection(electionName, candidateName) {
async.parallel(
{
voter: function(callback) {
Voter.findOne({ 'name' : candidateName }, function(err,voter) {
callback(err, voter);
});
}
},
function(e, r) {
if(r.voter === null){
return 'Voter not found';
} else {
Election.findOneAndUpdate(
{'name': electionName },
{$push: { candidates: r.voter }},
{new: true},
function(err, election) {
if(err){ return err; }
return (election) ? true : false;
});
}
}
);
}
I've already tried printing out the Voter instance(r.voter) to check if it exists (it does), and also printing out the election object returned by the mongoose call, which also works. However, I'm getting a null value in the
addedCandidatesSucessfully = _addCandidateToElection(electionName, candidates[i]);
line, regardless of the result of the call. I think it has to do with the mongoose call returning a local value which is never returned to the function that called _addCandidateToElection, but I don't know how I should return that. I've tried putting control flags such as
let foundAndUpdatedElection = false;
on the first line of _addCandidateToElection and updating it inside the Mongoose query's callback, but apparently it doesn't change.
How should I return the result of the query to the addCandidatesToElection function?
You should probably 'promisify' your code to help you better deal with the asynchronous nature of js. Try the following instead of your example:
function findVoter(candidateName) {
return new Promise(function(resolve, reject) {
Voter.findOne({ 'name' : candidateName }, function(err,voter) {
if(error) {
reject(error);
} else {
resolve(voter);
}
});
});
}
function addCandidateToElection(electionName, candidateName) {
return findVoter(candidateName).then(function(voter) {
return new Promise(function(resolve, reject) {
Election.findOneAndUpdate(
{'name': electionName },
{$push: { candidates: voter }},
{new: true},
function(err, election) {
if (err) {
reject(err);
} else {
resolve(!!election);
}
});
});
}
function addCandidatesToElection(req, res) {
let electionName = req.body.electionName;
let candidates = req.body.candidates;
let addedCandidatesSucessfully = true;
let candidatePromiseArray = [];
for(let i=0; i<candidates.length; i++) {
candidatePromiseArray.push(addCandidateToElection(electionName, candidates[i]));
}
Promise.all(candidatePromiseArray)
.then(function(results) {
console.log(results);
res.send('create election success');
})
.catch(function(error) {
console.error(error);
res.send('failed');
});
}
You will also no longer need to use the async library because promises are now native in ES6

node js mongo db dependencies (doc not being found)

I have the following code:
var method = PushLoop.prototype;
var agent = require('./_header')
var request = require('request');
var User = require('../models/user_model.js');
var Message = require('../models/message_model.js');
var async = require('async')
function PushLoop() {};
method.startPushLoop = function() {
getUserList()
function getUserList() {
User.find({}, function(err, users) {
if (err) throw err;
if (users.length > 0) {
getUserMessages(users)
} else {
setTimeout(getUserList, 3000)
}
});
}
function getUserMessages(users) {
// console.log("getUserMessages")
async.eachSeries(users, function (user, callback) {
var params = {
email: user.email,
pwd: user.password,
token: user.device_token
}
messageRequest(params)
callback();
}, function (err) {
if (err) {
console.log(err)
setTimeout(getUserList, 3000)
}
});
}
function messageRequest(params) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ params.email +"&api_password="+ params.pwd +"&method=getSMS&type=1&limit=5"
request(url, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
if (responseObject["status"] == "success") {
async.eachSeries(messages, function(message, callback){
console.log(params.token)
saveMessage(message, params.token)
callback();
}, function(err) {
if (err) {
console.log(err)
}
// setTimeout(getUserList, 3000)
})
} else {
// setTimeout(getUserList, 3000)
}
} else {
console.log(err)
// setTimeout(getUserList, 3000)
}
});
setTimeout(getUserList, 3000)
}
function saveMessage(message, token) {
// { $and: [ { price: { $ne: 1.99 } }, { price: { $exists: true } }
// Message.find({ $and: [{ message_id: message.id}, {device_token: token}]}, function (err, doc){
Message.findOne({message_id: message.id}, function (err, doc){
if (!doc) {
console.log('emtpy today')
var m = new Message({
message_id: message.id,
did: message.did,
contact: message.contact,
message: message.message,
date: message.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: token
});
m.save(function(e) {
if (e) {
console.log(e)
} else {
agent.createMessage()
.device(token)
.alert(message.message)
.set('contact', message.contact)
.set('did', message.did)
.set('id', message.id)
.set('date', message.date)
.set('message', message.message)
.send();
}
});
}
}) //.limit(1);
}
};
module.exports = PushLoop;
Which actually works perfectly fine in my development environment - However in production (i'm using Openshift) the mongo documents get saved in an endless loop so it looks like the (if (!doc)) condition always return true therefore the document gets created each time. Not sure if this could be a mongoose issue - I also tried the "find" method instead of "findOne". My dev env has node 0.12.7 and Openshift has 0.10.x - this could be the issue, and i'm still investigating - but if anybody can spot an error I cannot see in my logic/code please let me know
thanks!
I solved this issue by using a "series" like pattern and using the shift method on the users array. The mongoose upsert findOneOrCreate is good however if there is a found document, the document is returned, if one isn't found and therefore created, it's also returned. Therefore I could not distinguish between the newly insert doc vs. a found doc, so used the same findOne function which returns null if no doc is found I just create it and send the push notification. Still abit ugly, and I know I could have used promises or the async lib, might refactor in the future. This works for now
function PushLoop() {};
var results = [];
method.go = function() {
var userArr = [];
startLoop()
function startLoop() {
User.find({},function(err, users) {
if (err) throw err;
users.forEach(function(u) {
userArr.push(u)
})
function async(arg, callback) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ arg.email +"&api_password="+ arg.password +"&method=getSMS&type=1&limit=5"
request.get(url, {timeout: 30000}, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
var status = responseObject.status
if (status === "success") {
messages.forEach(function(m) {
var message = new Message({
message_id: m.id,
did: m.did,
contact: m.contact,
message: m.message,
date: m.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: arg.device_token
});
var query = { $and : [{message_id: m.id}, {device_token: arg.device_token}] }
var query1 = { message_id: m.id }
Message.findOne(query).lean().exec(function (err, doc){
if (!doc || doc == null) {
message.save(function(e) {
console.log("message saved")
if (e) {
console.log("there is an error")
console.log(e)
} else {
console.log(message.device_token)
var messageStringCleaned = message.message.toString().replace(/\\/g,"");
var payload = {
"contact" : message.contact,
"did" : message.did,
"id" : message.message_id,
"date" : message.date,
"message" : messageStringCleaned
}
var note = new apns.Notification();
var myDevice = new apns.Device(message.device_token);
note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now.
note.badge = 3;
note.alert = messageStringCleaned;
note.payload = payload;
apnsConnection.pushNotification(note, myDevice);
}
})
}
});
});
}
else {
console.log(err)
}
}
});
setTimeout(function() {
callback(arg + "testing 12");
}, 1000);
}
// Final task (same in all the examples)
function series(item) {
if(item) {
async( item, function(result) {
results.push(result);
return series(userArr.shift());
});
} else {
return final();
}
}
function final() {
console.log('Done');
startLoop();
}
series(userArr.shift())
});
}
}
module.exports = PushLoop;

Mongoose: MongoDB followers and following structure on update

I'm doing a following and followers addons to my code: here is the code.
In summary, the workflow is like this: I update an array of followers adding the ObjectId of the following person and vice versa,
however at the first update, my doc returns 0, it means that is hasnt been update, do you have any hint?
if (typeof(req.body.userToFollow) != 'undefined') {
var conditions = { _id: req.user._id }
, update = {"$addToSet":{following: req.body.userToFollow}}
, options = {};
Users.update(conditions, update, options, function(err, doc){
if (!err && doc) {
var reconditions = { _id: req.body.userToFollow }
, reupdate = {"$addToSet": { followers : req.user._id } }
, reoptions = {};
Users.update(conditions, update, options, function(err, doc){
if (!err && doc) {
var body = JSON.stringify(doc);
res.header('Content-Type', 'application/json');
res.end(body);
} else {
console.log(err);
}
});
} else {
console.log("\n\n\n\n\n");
console.log(doc);
}
});
}
many thanks
Use safe mode by passing {safe:true} in the options to update(). Then you will be able to check the status of the update.

Resources