Query with Mongoose multiple times without nesting - node.js

I'm trying to generate a document with node.js that needs to run multiple unrelated database queries from a mongo database.
Here is my current code:
Data.find({}, function(err, results) {
if (err) return next(err);
//finished getting data
res.render('page');
}
}
The problem is if I try to run another query, I seem to have to nest it within the first one so that it waits for the first one to finish before starting, and then I have to put res.render() within the innermost nested query (if I don't, res.render() will be called before the database is finished grabbing data, and it wont be rendered with the page).
What I have to do:
Data.find({}, function(err, results) {
if (err) return next(err);
//finished getting data
Data2.find({}, function(err, results2) {
if (err) return next(err);
//finished getting data 2
res.render('page');
}
}
}
}
I am going to have more than 2 queries, so if I keep nesting them it's going to get really messy really fast. Is there a cleaner way to do this, such as a way to make the code wait until all the data is returned and the function is run before continuing with the script?

For mongoose you can probably just do a Promise.all() and use .concat() on the resulting arrays of each query.
As a full demo:
var async = require('async'),
mongoose = require('mongoose'),
Schema = mongoose.Schema;
var d1Schema = new Schema({ "name": String });
var Data1 = mongoose.model("Data1", d1Schema);
var d2Schema = new Schema({ "title": String });
var Data2 = mongoose.model("Data2", d2Schema);
mongoose.set('debug',true);
mongoose.connect('mongodb://localhost/test');
async.series(
[
// Clean
function(callback) {
async.each([Data1,Data2],function(model,callback) {
model.remove({},callback)
},callback);
},
// Setup some data
function(callback) {
async.each([
{ "name": "Bill", "model": "Data1" },
{ "title": "Something", "model": "Data2" }
],function(data,callback) {
var model = data.model;
delete data.model;
mongoose.model(model).create(data,callback);
},callback);
},
// Actual Promise.all demo
function(callback) {
Promise.all([
Data1.find().exec(),
Data2.find().exec()
]).then(function(result) {
console.log([].concat.apply([],result));
callback()
}).catch(callback);
}
],
function(err) {
if (err) throw err;
mongoose.disconnect();
}
)
I'm just mixing in async there for brevity of example, but the meat of it is in:
Promise.all([
Data1.find().exec(),
Data2.find().exec()
]).then(function(result) {
console.log([].concat.apply([],result));
})
Where the Promise.all() basically waits for and combines the two results, which would be an "array of arrays" here but the .concat() takes care of that. The result will be:
[
{ _id: 59420fd33d48fa0a490247c8, name: 'Bill', __v: 0 },
{ _id: 59420fd43d48fa0a490247c9, title: 'Something', __v: 0 }
]
Showing the objects from each collection, joined together in one array.
You could also use the async.concat method as an alternate, but unless you are using the library already then it's probably just best to stick to promises.

Related

Modify mongoose response data

I have a an API call that returns all messages a user has recieved as JSON.
The model data looks something like this:
{
sender: ObjectId,
reciever: ObjectId,
message: String
}
What is the proper way to modify the JSON the API responds with?
I want to end up with data grouped like so:
{
<senderid>:[ all of the messages from this sender],
<other_sender>:[ all of the messages from this sender]
}
Do I have to manually do this in javascript, or is there a faster way to do this taking advantage of mongoose?
Using the aggregation framework will be ideal for this task. You could run the following aggregation pipeline that makes use of the $group operator step to group the data to process them. The group pipeline operator is similar to the SQL's GROUP BY clause. In SQL, you can't use GROUP BY unless you use any of the aggregation functions. The same way, you have to use an aggregation function in MongoDB as well. In this instance, use the $push accumulator operator to create the array of messages.
Since Model.aggregate() returns plain objects you would then transform the resulting array to the desired hash key using lodash library's _.indexBy() method:
var pipeline = [
{
"$group": {
"_id": "$sender",
"messages": { "$push": "$message" }
}
}
];
Model.aggregate(pipeline,
function(err, res) {
if (err) return handleError(err);
var hashmap = _.chain(res)
.indexBy('_id')
.mapValues('messages')
.value();
console.log(JSON.stringify(hashmap, undefined, 4));
}
);
// Or use the aggregation pipeline builder.
Model.aggregate()
.group({ "_id": "$sender", "messages": { "$push": "$message" } })
.exec(function (err, res) {
if (err) return handleError(err);
var hashmap = _.chain(res)
.indexBy('_id')
.mapValues('messages')
.value();
console.log(JSON.stringify(hashmap, undefined, 4));
});
Check the demo below.
var data = [
{ _id: 'user1', messages: ['msg1', 'msg2'] },
{ _id: 'user2', messages: ['msg3', 'msg1'] },
{ _id: 'user3', messages: ['msg6', 'msg3'] },
{ _id: 'user4', messages: ['msg4', 'msg8'] }
];
var hashmap = _.chain(data)
.indexBy('_id')
.mapValues('messages')
.tap(log)
.value();
function log(value) {
pre.innerHTML += JSON.stringify(value, null, 4) + "\n"
}
<script src="//cdnjs.cloudflare.com/ajax/libs/lodash.js/3.10.1/lodash.min.js"></script>
<pre id="pre"></pre>

Get total count along with Mongoose Query skip & limt

I have a json data which contains many objects. I want to limit the data for pagination and I need the total items count. Please help.
Model.find().skip((pageNumber-1)*limit).limit(limit).exec()
I want the count and skipped data in response.
You can use async library for running 2 queries at once. In your case you can run one query to get the number of documents and another for pagination.
Example with 'User' model:
var async = require('async');
var User = require('./models/user');
var countQuery = function(callback){
User.count({}, function(err, count){
if(err){ callback(err, null) }
else{
callback(null, count);
}
}
};
var retrieveQuery = function(callback){
User.find({}).skip((page-1)*PAGE_LIMIT)
.limit(PAGE_LIMIT)
.exec(function(err, doc){
if(err){ callback(err, null) }
else{
callback(null, doc);
}
}
};
async.parallel([countQuery, retrieveQuery], function(err, results){
//err contains the array of error of all the functions
//results contains an array of all the results
//results[0] will contain value of doc.length from countQuery function
//results[1] will contain doc of retrieveQuery function
//You can send the results as
res.json({users: results[1], pageLimit: PAGE_LIMIT, page: page, totalCount: results[0]});
});
async allows you to run a number of queries in parallel depending on the hardware you are using. This would be faster than using 2 independent queries to get count and get the required documents.
Hope this helps.
I have solved it with $facet and aggregate the following way in mongoose v3+:
const [{ paginatedResult, [{ totalCount }] }] = await Model.aggregate([{
$facet: {
paginatedResult: [
{ $match: query },
{ $skip: skip },
{ $limit: limit }
],
totalCount: [
{ $match: query },
{ $count: 'totalCount' }
]
}
}])
where the totalCount refers the total number of records matching the search query while the paginatedResult is only the paginated slice of them.
The problem with these solutions is that for every request you are doing two queries. This becomes problematic when you have a complex data structure and large data set as performance becomes an issue. Consider instead creating a special function that listens for the /resource?count=true or /resource/count GET methods and returns only the count.
You need to perform 2 queries to achieve that. One to get results and another to get total items amount with .count().
For example code you can watch at on of "paginator" for mongoose mongoose-paginate.
To performe only one query, you may use the find() method associated with promises and array slices. A small example would be:
getPaginated(query, skip, limit){
return this.model.find(query)
.lean()
.then((value)=>{
if (value.length === 0) return {userMessage: 'Document not found'};
const count = value.length;
//skip===0 must be handled
const start = parseInt(limit)*parseInt(skip - 1);
const end = start + parseInt(reqQuery.pagesize);
//slicing the array
value = value.slice(start,end);
//could return it another way...
value.push( { 'querySize': count });
return value;
})
.catch((reason)=>{
//...handling code
});
}

Mongoose: Insert multiple documents and then disconnect database?

When using Mongoose to insert multiple documents into a collection, since each .save() method has it's own callback, how do you know when they are all complete so that you can mongoose.disconnect()?
Lets say I have 3 documents I need to insert into the database:
var database = mongoose.connect('mongodb://localhost/somedb');
document1.create({...}, function(err){
if (err) { ... }
// It's saved!
});
document2.create({...}, function(err){
if (err) { ... }
// It's saved!
});
document3.create({...}, function(err){
if (err) { ... }
// It's saved!
});
database.disconnect();
The disconnect is most likely going to happen before the documents get saved to the database, especially if the Mongodb server is remote or slow or something.
What is the best way to handle this? Are Promises the only way? What was the solution to this a few years ago before Promises were so prevalent?
One option instead of using promises is to use the async library that has been around for some years now. There are nice helpers such as async.series and async.parallel that provide a final function call when all provided methods are complete, with the difference there being whether you need the operations to complete in series ( one after the other ) on simply just execute all together in parallel:
async.parallel(
[
function(callback) {
document1.create({...},callback);
},
function(callback) {
document2.create({...},callback);
},
function(callback) {
document3.create({...},callback);
},
],
// Called when all completed or on an error
function(err,results) {
// err is any error
// results contains an array ( in this case ) of any results
mongoose.disconnect();
}
)
Since the first argument in either case is either an array or an object containing the function calls, then you can just build up said array/object and pass it in, as long as the second argument is the handler that recieves any error or otherwise executes when all are complete.
So you can for example "build" like this:
var list = [
{ "model": "Model1", "data": {...} },
{ "model": "Model2", "data": {...} },
{ "model": "Model3", "data": {...} }
];
list = list.map(function(item) {
return function(callback) {
mongoose.model(item.model).create(item.data,callback);
}
});
async.parallel(list,function(err,results) {
// err is any error
// results contains an array ( in this case ) of any results
mongoose.disconnect();
});
And that all works the same as if hardcoded, but in a simple and programatic way.
Of course the other natural approach is to simply nest each call within their callbacks, but libraries such as shown are meant to make things prettier and easier to manipulate for creation of such a list.

How to implement map function of Mongodb cursor in node.js (node-mondodb-native)

I am trying to implement following MongoDB query in NodeJS
db.tvseries.find({}).map(function(doc){
var userHasSubscribed = false;
doc.followers && doc.followers.forEach(function(follower) {
if(follower.$id == "abc") {
userHasSubscribed = true;
}
});
var followers = doc.followers && doc.followers.map(function(follower) {
var followerObj;
db[follower.$ref].find({
"_id" : follower.$id
}).map(function(userObj) {
followerObj = userObj;
});
return followerObj;
});
return {
"id": doc.name,
"userHasSubscribed": userHasSubscribed,
"followers": followers || []
};
})
Following is the db
users collection
{
"id": ObjectId("abc"),
"name": "abc_name"
},
{
"id": ObjectId("def"),
"name": "def_name"
},
{
"id": ObjectId("ijk"),
"name": "ijk_name"
}
tvseries collection
{
"id": ObjectId("123"),
"name": "123_name",
"followers": [
{
"$ref": "users",
"$id": ObjectId("abc"),
},
{
"$ref": "users",
"$id": ObjectId("def"),
}
]
},
{
"id": ObjectId("456"),
"name": "456_name",
"followers": [
{
"$ref": "users",
"$id": ObjectId("ijk"),
},
]
},
{
"id": ObjectId("789"),
"name": "789_name"
}
I am not able to figure out how to execute the above MongoDB query in NodeJS with the help of node-mongodb-native plugin.
I tried the below code but then I get TypeError: undefined is not a function at .map
var collection = db.collection('users');
collection.find({}).map(function(doc) {
console.log(doc);
});
How to execute .map function in NodeJS?
Thanks in advance
I struggled with this for some time. I found that by adding .toArray() after the map function works.
You could even skip map and only add .toArray() to get all the documents fields.
const accounts = await _db
.collection('accounts')
.find()
.map(v => v._id) // leaving this out gets you all the fields
.toArray();
console.log(accounts); // [{_id: xxx}, {_id: xxx} ...]
Please take note that in order for map to work the function used must return something - your example only console.logs without returning a value.
The forEach solution works but I really wanted map to work.
I know that I'm pretty late but I've arrived here by searching on Google about the same problem. Finally, I wasn't able to use map function to do it, but using forEach did the trick.
An example using ES6 and StandardJS.
let ids = []
let PublicationId = ObjectID(id)
feeds_collection
.find({PublicationId})
.project({ _id: 1 })
.forEach((feed) => {
ids.push(feed._id)
}, () => done(ids))
To echo #bamse's anwer, I got it working with .toArray(). Here is an async example:
async function getWordArray (query) {
const client = await MongoClient.connect(url)
const collection = client.db('personal').collection('wordBank')
const data = await collection.find(query).map(doc => doc.word).toArray()
return data
}
Then I use it in my Express route like this:
app.get('/search/:fragment', asyncMiddleware(async (req, res, next) => {
const result = await getWordArray({word: 'boat'})
res.json(result)
}))
Finally, if you need a guide to async/await middleware in NodeJS, here is a guide: https://medium.com/#Abazhenov/using-async-await-in-express-with-node-8-b8af872c0016
map returns a cursor, toArray returns a Promise that will execute a cursor and return it's results. That may be an array of the original query find, limit etc. or a promise of an array of those result piped through a function.
This is typically useful when you want to take the documents of the cursor and process that (maybe fetch something else) while the cursor is still fetching documents, as opposed to waiting until they have all been fetched to node memory
Consider the example
let foos = await db.collection("foos")
.find()
.project({
barId: 1
})
.toArray() // returns a Promise<{barId: ObjectId}[]>
// we now have all foos into memory, time to get bars
let bars = await Promise.all(foos.map(doc => db
.collection("bars")
.findOne({
_id: doc.barId
})))
this is roughly equivalent to
bars = await db.collection("foos")
.find()
.project({
barId: 1
})
.toArray() // returns a Promise<{barId: ObjectId}[]>
.then(docs => docs
.map(doc => db
.collection("bars")
.findOne({
_id: doc.barId
})))
using map you can perform the operation asynchrounsly and (hopefully) more efficiently
bars = await db.collection("foos")
.find()
.project({
barId: 1
})
.map(doc => db
.collection("bars")
.findOne({
_id: doc.barId
}))
.toArray()
.then(barPromises => Promise.all(barPromises)) // Promise<Bar[]>
The main point is that map is simply a function to be applied to the results fetched by the cursor. That function won't get executed until you turn it into a Promise, using either forEach or more sensibly, map

MongoDB and NodeJS get related data from 3 collections

i have a mongoDB query to get data with $group and $count.
This data contains the _id from other documents collection.
How can i get the other documents by its _id in NodeJS and MongoDB asyncrohnous?
db.orders.aggregate([
{$match: { 'works.TechnicianId': {$in:['53465f9d519c94680327965d','5383577a994be8b9a9e3f01e']},
'works.Date': {$gte: ISODate("2013-05-21T06:40:20.299Z"), $lt: ISODate("2016-05-21T06:40:20.299Z")}}},
{$unwind: "$works" },
{$group: {_id: "$works.TechnicianId",total:{$sum:'$works.price'},ordersId: { $push: "$_id" }}},
])
This is the result:
{
"result" : [
{
"_id" : "53465f9d519c94680327965d",
"total" : 198,
"ordersId" : [
ObjectId("537b5ea4c61b1d1743f4341f"),
ObjectId("537b4633021d75bd36863f29")
]
},
{
"_id" : "5383577a994be8b9a9e3f01e",
"total" : 22,
"ordersId" : [
ObjectId("537b5ea4c61b1d1743f4341f"),
ObjectId("537b4633021d75bd36863f29")
]
}
],
"ok" : 1
}
Now i need to get from orders collection the documents with id from ordersId, and from other collection the documents with _id from the result _id field.
I try with this:
var collection = db.collection('orders');
var result = [];
collection.aggregate([
{
$match: {
'works.TechnicianId': {
$in: ids
},
'works.Date': {
$gte: new Date(startDate),
$lt: new Date(endDate)
}
}
},
{
$unwind: "$works"
},
{
$group: {
_id: "$works.TechnicianId",
total: {
$sum: '$works.price'
},
orderId: {
$push: "$_id"
}
}
}
],
function (e, docs) {
if (e) {
error(e);
}
var usersCollection = db.collection('users');
_.each(docs, function (doc) {
usersCollection.findOne({_id: new ObjectID(doc._id)}, function (e, doc) {
doc.tech = doc;
});
doc.orders = [];
_.each(doc.orderId, function (queryOrder) {
collection.findOne({_id: new ObjectID(queryOrder._id)}, function (e, order) {
doc.orders.push(order);
});
});
success(docs);
});
});
But the success its called before all the _.eachs are finished..Any help, or idea?
Edit:
I try with Q promises, this is my code:
var usersCollection = db.collection('users');
var promises = [];
_.each(reports, function (report) {
var promise = usersCollection.findOne({_id: new ObjectID(report._id)}).then(
function (e, orderUserReported) {
if (e) {
error(e);
}
report.tech = orderUserReported;
_.each(orderUserReported.orderId, function (queryOrder) {
collection.findOne({_id: new ObjectID(queryOrder._id)}, function (e, order) {
report.orders.push(order);
});
});
});
promises.push(promise);
});
Q.allSettled(promises).then(success(reports));
and the error:
/Users/colymore/virteu/aa/server/node_modules/mongodb/lib/mongodb/connection/base.js:245
throw message;
^
TypeError: Cannot call method 'then' of undefined
Because of asynchronous execution you have to wait until results are returned. There are several options available:
async library https://github.com/caolan/async
promises https://github.com/kriskowal/q
Async is closer to your current code, you could use async.parallel https://github.com/caolan/async#parallel to wait untill you get data back
Update
Mongoose functions don't return Q promises, so you need to convert mongoose calls to promises by using something like Q.denodeify(User.findOne.bind(models.User))({ _id: userId}).then(...
For your case Q.denodeify(userCollection.findOne.bind(userCollection))({_id: new ObjectID(report._id)}).then(...
Short answer: Use promises. Look at Q.allSettled ( https://github.com/kriskowal/q )
Just run success asynchronously when all subtask are done.
Also using https://github.com/iolo/mongoose-q package may be helpful to not combine mongoose promises with Q ones if you want use mongoose in your mongo.

Resources