Making butch upsert in mongodb: callback never fired - node.js

I have an array of documents with unique _id and I want to insert them to my database. Some of them already in db, and for those I want to update an array property (push in array an item). All of this I need to make asyncronuosly, so after all inserted/updated I want to write response back (with callback) to client than all ok or write an error. After googling on subject I've found this solution with async module I've tried to implement it for my case. Now my code looks like this:
function processUsers(arr, listName, callback) {
var users = global.db.collection('vkusers');
var q = async.queue(function(task, cb) {
console.log('upsert butch');
users.insert(task.doc, function(err, doc) {
if (err) {
users.update({
_id : task.doc._id
}, {
$addToSet : {
mLists : listName
}
}, function(error, result){ console.log(error); console.log(result); });
}
});
}, arr.length);
for ( var doc in arr) {
q.push({
doc : arr[doc]
}, function(err) {
if (err)
callback(err, null);
})
}
q.drain = function() {
// this is the queue's callback, called when the queue is empty,
// i.e. when all your documents have been processed.
console.log('drain');
callback(null, { result: "success", upserted: arr.length });
}
}
Callback has signature callback(error, result), arr - my array of documents. I've tested it and with database everything is OK, i am getting the right result. But callback, and q.drain never fired!

You need to call async.queue's callback (cb in your code) when your insert/update is complete. Something like this:
var q = async.queue(function(task, cb) {
console.log('upsert butch');
users.insert(task.doc, function(err, doc) {
if (err) {
users.update({
_id : task.doc._id
}, {
$addToSet : {
mLists : listName
}
}, function(error, result) {
console.log(error);
console.log(result);
cb(error); // Update finished; call cb and pass in "error" so that it can bubble up if it exists
});
} else {
cb(); // Insert succeeded; call cb
}
});
}, arr.length);

Related

Mongoose find() and append to array

I am trying to asynchronously retrieve some data from a database and then append those values in an array I need for some other things, I have this function:
function retrieveProduct(SKU, callback) {
Products.find({
SKU: key
}, function (err, doc) {
if (err) {
callback(err, null);
} else {
callback(null, doc[0]);
}
});
}
and I use it like this:
for (var key in orders) {
retrieveProduct(key, function (err, doc) {
if (err) {
console.log(err);
}
products_found[key] = doc
console.log(doc);
});
}
console.log(products_found);
}
Where orders is a list of IDs.
The problem is that when I try to append doc to products_found nothing is being appended and nothing is logged to the console. Moreover, ESlint is telling me not to make functions within a loop.
What am I doing wrong?

How to get data from one collection and insert into another collection in Nodejs?

Am using Nodejs and MongoDB and I am new to nodejs. I need to know how to get data from one collection and append some additional data and insert into another collection.
db.collection('collection1').find({ "Id" : 12345 }).toArray(function(err, result){
db.collection('collection2', function(err, collection){
collection.insert({
//some data
})
})
})
When I try this code its not working its giving me error insert is not defined.
thanks,
John.
db.collection('collection1').find({ "Id" : 12345 }).toArray(function(err, result){
//do the modification here
db.collection('collection2').insert(modifiedResult, function(err, result){
if(err) {
//log error
}else{
//log result
}
})
})
One more thing, If the result array length is more that one and you want to insert then separately then use promise
db.collection('collection1').find({ "Id" : 12345 }).toArray(function(err, result){
//do the modification here
Promise.all(modifiedResult.map((eachModifiedResult)=>{
return db.collection('collection2').insert(eachModifiedResult);
}).then((result)=>{
//result of the insert
}).catch((err){
//err if any happen
});
})
But if you have a very large doc then do it as Neil Said. Read the collection one by one using cursor and modify them and insert them to other db.
You can use callback library like async or Promises Q
Promise
var collectionData = null;
var modifiedResult = null;
// here i am using async library to avoid callbackHell
async.series([
// for get data from collection 1.
function(cb) {
var criteria = {
"Id": 12345
}
db.collection('collection1').find(criteria).toArray(function(dbErr, dbResult) {
if (err) {
cb(dbErr)
} else {
collectionData = dbResult;
cb()
}
})
},
// Append Data in collectionData
function(cb) {
// do you work here to append data in collectionData
modifiedResult = extendedData; // this is just an example you need to work on it
cb();
},
// Update collection 2 here
function(cb) {
db.collection('collection2').insert(modifiedResult, function(err, result) {
if (err) {
cb(dbErr)
} else {
collectionData = dbResult;
cb()
}
});
}
]);

update field in node js

I want to update the plateNumber in my table car.
I used this for a specific ID:
car.update({_id:'436' }, {$set: {plateNumber: 'DG-884-AQ'}}, {multi: true},cb);
But It didn't work, and I got
ReferenceError: cb is not defined
What should I do?
and I use car as instance or Car as the model.
Thanks.
You need to provide a function:
car.update(
{_id:'436' },
{$set: {plateNumber: 'DG-884-AQ'}},
{multi: true},
(err, data) => {
if (err) {
// handle error
} else {
// handle success
}
}
);
Your code would work if you had the cb function defined with:
let cb = (err, data) => {
if (err) {
// handle error
} else {
// handle success
}
};
or:
function cb(err, data) {
if (err) {
// handle error
} else {
// handle success
}
}
but you cannot pass cb as an argument if you don't have it defined.
cb is suppose to be a function(callback)
car.update(
{_id:'436' },
{$set: {plateNumber: 'DG-884-AQ'}},
{multi: true}, function(err, resp) {
});

Node async callback was already called when trying to make a nested query

I am getting a Callback was already called error while trying to make asynchronous queries using the MEAN stack. I need the second callback to only trigger after the nested query has been completed (as per the comments in the code). Why am I getting this error?
Example of the route:
router.route('/teams/:user_id').get(function (req, res) {
TeamProfile.find({
Members : {
$in : [req.params.user_id]
}
}).exec(function (err, teamProfiles) {
var asyncTasks = [];
teamProfiles.forEach(function (teamProfile) {
asyncTasks.push(function (callback) {
UserProfile.find({
UserID : {
$in : teamProfile.Members.map(function (id) {
return id;
})
}
}, function (err, userProfiles) {
teamProfile.Members = userProfiles;
callback();
})
});
});
teamProfiles.forEach(function (teamProfile) {
asyncTasks.push(function (callback) {
Draft.find({
_id : {
$in : teamProfile.Drafts.map(function (id) {
return id;
})
}
}, function (err, drafts) {
teamProfile.Drafts = drafts;
drafts.forEach(function (draft) {
Comment.find({
_id : {
$in : draft.Comments.map(function (id) {
return id;
})
}
}).exec(function (err, comments) {
draft.Comments = comments;
callback();
//This is where the callback should be called
//Throws Error: Callback was already called.
})
})
})
});
});
async.parallel(asyncTasks, function () {
res.json(teamProfiles)
});
});
})
I am using async.parallel() to perform all the queries. I am very new to all of this so please excuse some beginner mistakes.
You are iterating over drafts synchronously and calling async's callback function on the first item. Getting an error when you try to call it again with the second item is expected behaviour.
You should instead call the done callback once you have finished all the draft queries, not for each one. Since you are using async, you could nest another async.each to handle this:
router.route('/teams/:user_id').get(function (req, res) {
TeamProfile.find({
Members : {
$in : [req.params.user_id]
}
}).exec(function (err, teamProfiles) {
var asyncTasks = [];
teamProfiles.forEach(function (teamProfile) {
asyncTasks.push(function (callback) {
UserProfile.find({
UserID : {
$in : teamProfile.Members.map(function (id) {
return id;
})
}
}, function (err, userProfiles) {
teamProfile.Members = userProfiles;
callback();
});
});
});
teamProfiles.forEach(function (teamProfile) {
asyncTasks.push(function (callback) {
Draft.find({
_id : {
$in : teamProfile.Drafts.map(function (id) {
return id;
})
}
}, function (err, drafts) {
teamProfile.Drafts = drafts;
async.each(drafts, function(draft, draftCallback){
Comment.find({
_id : {
$in : draft.Comments.map(function (id) {
return id;
})
}
}).exec(function (err, comments) {
draft.Comments = comments;
draftCallback();
});
}, function(err){
callback();
});
});
});
});
async.parallel(asyncTasks, function () {
res.json(teamProfiles)
});
});
});

MongoJS synchronous call

I'm fixing some bugs in one project. I need synchronous cursor, to load data in loop.
I have code:
var mongo = require('mongojs'),
dataMaps = [];
mongo.penguin_friends.find({ user_id_1: id_user }, function(err, friends) {
friends.forEach(function(data) {
var cursor = mongo.penguin_user_maps3.find({ user_id: data.id_user }, { fields: { maps: 1 } });
//I need to do dataMaps.push(cursor.data);
});
console.log("processingThisSomething()");
processSomething();
});
I need to complete this request before calling processSomething(); So I need to process mongodb query inside a loop synchronously.
It's not possible to make the queries synchronous as the API doesn't support it.
You'll have to provide a callback to .find() or a cursor method to receive the results:
cursor.toArray(function (err, maps) {
if (!err) {
dataMaps.push(maps);
}
});
But, you can replace the iterator with one that's asynchronous-aware, such as async.each(), to continue when they've completed:
async.each(
friends,
function (data, callback) {
var cursor = mongo....;
cursor.toArray(function (err, maps) {
if (!err) {
dataMaps.push(maps);
}
callback(err);
});
},
function (err) {
if (!err) {
processSomething();
}
}
);

Resources