Modify mongoose response data - node.js

I have a an API call that returns all messages a user has recieved as JSON.
The model data looks something like this:
{
sender: ObjectId,
reciever: ObjectId,
message: String
}
What is the proper way to modify the JSON the API responds with?
I want to end up with data grouped like so:
{
<senderid>:[ all of the messages from this sender],
<other_sender>:[ all of the messages from this sender]
}
Do I have to manually do this in javascript, or is there a faster way to do this taking advantage of mongoose?

Using the aggregation framework will be ideal for this task. You could run the following aggregation pipeline that makes use of the $group operator step to group the data to process them. The group pipeline operator is similar to the SQL's GROUP BY clause. In SQL, you can't use GROUP BY unless you use any of the aggregation functions. The same way, you have to use an aggregation function in MongoDB as well. In this instance, use the $push accumulator operator to create the array of messages.
Since Model.aggregate() returns plain objects you would then transform the resulting array to the desired hash key using lodash library's _.indexBy() method:
var pipeline = [
{
"$group": {
"_id": "$sender",
"messages": { "$push": "$message" }
}
}
];
Model.aggregate(pipeline,
function(err, res) {
if (err) return handleError(err);
var hashmap = _.chain(res)
.indexBy('_id')
.mapValues('messages')
.value();
console.log(JSON.stringify(hashmap, undefined, 4));
}
);
// Or use the aggregation pipeline builder.
Model.aggregate()
.group({ "_id": "$sender", "messages": { "$push": "$message" } })
.exec(function (err, res) {
if (err) return handleError(err);
var hashmap = _.chain(res)
.indexBy('_id')
.mapValues('messages')
.value();
console.log(JSON.stringify(hashmap, undefined, 4));
});
Check the demo below.
var data = [
{ _id: 'user1', messages: ['msg1', 'msg2'] },
{ _id: 'user2', messages: ['msg3', 'msg1'] },
{ _id: 'user3', messages: ['msg6', 'msg3'] },
{ _id: 'user4', messages: ['msg4', 'msg8'] }
];
var hashmap = _.chain(data)
.indexBy('_id')
.mapValues('messages')
.tap(log)
.value();
function log(value) {
pre.innerHTML += JSON.stringify(value, null, 4) + "\n"
}
<script src="//cdnjs.cloudflare.com/ajax/libs/lodash.js/3.10.1/lodash.min.js"></script>
<pre id="pre"></pre>

Related

MongoDB Query Optimize Search Text

I have an application developed in NodeJS, which works as a REST API and consumes data from MongoDB
In MongoDB I have a collection called 'ftp' with more than 10 million documents with the following structure
{
"_id" : ObjectId("59e7c66911506bd1725cf145"),
"ip" : "72.32.177.76",
"timestamp" : "2017-10-16T02:30:26-04:00",
"data" : {
"banner" : "220-FileZilla Server version 0.9.41 beta\r\n"
}
}
The "data.banner" field is a hased index
From NoodeJs I make an aggregate query that filters a string of text using a regular expression, groups and counts the results.
function getData(timeInit, table, textSearch, res) {
MongoClient.connect(url, function (err, db) {
if (err) throw err;
db.collection(table).aggregate([
{
$match: { 'data.banner': $regex:textSearch}
}, {
$group: {
_id: '$data.banner',
num: { $sum: 1 },
}
},
{
$sort: {
num: -1
}
},{
$limit:5
}
], {
allowDiskUse: true
}
).toArray(function (err, result) {
if (err) throw err;
var timeFinal = new Date();
var data = {
result: result,
timeLapse: (timeFinal - timeInit) / 1000,
numResult: result.length
};
res.send(data);
db.close();
});
});
};
The query with regular expression takes about 8 seconds to return results, an excessive time in my opinion, since the regular expressions are not optimal.
My question is how should I make the filter to search for documents that contain text in an optimal way reducing the response time.
If someone knows how to optimize this type of query I would appreciate it a lot.

Query with Mongoose multiple times without nesting

I'm trying to generate a document with node.js that needs to run multiple unrelated database queries from a mongo database.
Here is my current code:
Data.find({}, function(err, results) {
if (err) return next(err);
//finished getting data
res.render('page');
}
}
The problem is if I try to run another query, I seem to have to nest it within the first one so that it waits for the first one to finish before starting, and then I have to put res.render() within the innermost nested query (if I don't, res.render() will be called before the database is finished grabbing data, and it wont be rendered with the page).
What I have to do:
Data.find({}, function(err, results) {
if (err) return next(err);
//finished getting data
Data2.find({}, function(err, results2) {
if (err) return next(err);
//finished getting data 2
res.render('page');
}
}
}
}
I am going to have more than 2 queries, so if I keep nesting them it's going to get really messy really fast. Is there a cleaner way to do this, such as a way to make the code wait until all the data is returned and the function is run before continuing with the script?
For mongoose you can probably just do a Promise.all() and use .concat() on the resulting arrays of each query.
As a full demo:
var async = require('async'),
mongoose = require('mongoose'),
Schema = mongoose.Schema;
var d1Schema = new Schema({ "name": String });
var Data1 = mongoose.model("Data1", d1Schema);
var d2Schema = new Schema({ "title": String });
var Data2 = mongoose.model("Data2", d2Schema);
mongoose.set('debug',true);
mongoose.connect('mongodb://localhost/test');
async.series(
[
// Clean
function(callback) {
async.each([Data1,Data2],function(model,callback) {
model.remove({},callback)
},callback);
},
// Setup some data
function(callback) {
async.each([
{ "name": "Bill", "model": "Data1" },
{ "title": "Something", "model": "Data2" }
],function(data,callback) {
var model = data.model;
delete data.model;
mongoose.model(model).create(data,callback);
},callback);
},
// Actual Promise.all demo
function(callback) {
Promise.all([
Data1.find().exec(),
Data2.find().exec()
]).then(function(result) {
console.log([].concat.apply([],result));
callback()
}).catch(callback);
}
],
function(err) {
if (err) throw err;
mongoose.disconnect();
}
)
I'm just mixing in async there for brevity of example, but the meat of it is in:
Promise.all([
Data1.find().exec(),
Data2.find().exec()
]).then(function(result) {
console.log([].concat.apply([],result));
})
Where the Promise.all() basically waits for and combines the two results, which would be an "array of arrays" here but the .concat() takes care of that. The result will be:
[
{ _id: 59420fd33d48fa0a490247c8, name: 'Bill', __v: 0 },
{ _id: 59420fd43d48fa0a490247c9, title: 'Something', __v: 0 }
]
Showing the objects from each collection, joined together in one array.
You could also use the async.concat method as an alternate, but unless you are using the library already then it's probably just best to stick to promises.

Is it possible to get count of number of docs returned from find() query in mongoose

I am trying to get count of data fetched from the database using find() query in mongoose. Now can anyone tell me can i do something like below or do i have to write other function to do that
merchantmodel.find({merchant_id: merchant_id, rating: {'$ne': -1 }, review: {'$ne': "" }}, {'review':1, '_id':0}, {sort: {time_at: -1}}, function(err, docs) {
if (err) {
} else {
if (docs) {
console.log(docs[1].review);
console.log(docs.size()); // Here by writing something is it possible to get count or not
res.json({success: 1, message : "Successfully Fetched the Reviews"});
}
}
});
Convert returned value to array and then use length property
var query = { merchant_id : merchant_id, rating : { '$ne': -1 }, review: { '$ne': "" }};
var projection = { 'review':1, '_id':0 };
var options = { sort: { time_at: -1 } };
merchantmodel.find(query, projection, options).toArray(function(err, docs) {
if (err) {
throw(err);
}
console.log(docs[1].review);
console.log(docs.length);
res.json({success: 1, message : "Successfully Fetched the Reviews"});
});
You can simply do this:
console.log(docs.length);
The docs variable returned by the find() method is an array so docs.length would do the job.
The mongodb native way to do this would be:
db.collection.find( { a: 5, b: 5 } ).count()

Many-to-Many in MongoDB

I have two MongoDB models "Users" and "Roles". Each user can have multiple roles and each role can be assigned to many users. To keep the relation simple I would like to store the reference between both models only in the "Users" model which is already working as expected. But when I'm loading all roles at once with .find({}), I would also like to know how many users are assigned to these roles (to check if a role can be modified).
I'm using Node.js + ExpressJS and mongoose. This is what I already have:
var userSchema = new Schema({
username: String,
...
roles : [ {
type : Number,
ref : 'Role'
} ]
});
var roleSchema = new Schema({
_id : Number,
name : String
});
--------------------------------------------------
function getRoles(request, response) {
Role.find({}, function(err, roles) {
....
response.send(roles);
});
}
But now I wonder how I would achive the count query per role and still be able to send the result in one response.
I would like to avoid to send a single request per role and try to do the count within the getRoles() function. In a relational database I would do something like this:
select r.*,
(select count(*) from user2role u2r where u2r.role_id = r.role_id)
from role r;
But what's the MongoDB equivalent?
Any help and hints will be appreciated.
Thanks, Gerry
To achieve the user count query per role with the given schema, you could either use the count() method as follows:
function getUsersCount(roleName, req, res) {
Role.findOne({"name": roleName}, function(err, role) {
var roleId = role._id;
// var countQuery = User.where({ "role": roleId }).count();
// User.count({ "roles": roleId }).count(callback)
// User.count({ "roles": roleId }, callback)
User.where({ "roles": roleId }).count(function (err, count) {
if (err) return handleError(err);
console.log("There are %d users with the role %d", count, roleName);
res.send(count);
})
});
}
or you can use the aggregate() method like:
// Find the user count for a given role
function getUsersCount(roleName, req, res) {
Role.findOne({"name": roleName}, function(err, role) {
var roleId = role._id;
var pipeline = [
{
"$match": { "roles": roleId } /* filter the documents that only have the roleId in the role array */
},
{ "$unwind": "$roles" },
{
"$match": { "roles": roleId }
},
{
"$group": {
"_id": null, "count": { "$sum": 1 }
}
},
{
"$project": {
"_id": 0, "count": 1
}
}
];
Users.aggregate(pipeline, function (err, result) {
if (err) return handleError(err);
console.log(result); // [ { count: 55 } ] for example
res.send(result);
});
// Or use the aggregation pipeline builder.
Users.aggregate()
.match({ "roles": roleId })
.unwind("roles")
.match({ "roles": roleId })
.group({ "_id": null, "count": { "$sum": 1 } })
.select("-id count")
.exec(function (err, result) {
if (err) return handleError(err);
console.log(result); // [ { count: 55 } ] for example
res.send(result);
});
});
}
-- EDIT --
One approach that you could take is to use the lean() method to get pure JavaScript objects that you can manipulate to add the extra userCount field for each role. With lean() it's possible because documents returned from queries with the lean option enabled are plain javascript objects', not MongooseDocuments. Thus with the roles returned from the find() cursor, you can convert them into plain JS objects
that you can iterate over using the native JavaScript map() method, get the number of users per role using another query that takes into consideration the count() method explained above and return the modified object.
The following demonstrates how the function is implemented:
function getRoles(request, response) {
Role.find({}).lean().exec(function(err, roles) {
var result = roles.map(function(r){
var obj = {},
countQuery = User.where({ "roles": r._id }).count();
obj["userCount"] = countQuery;
obj["_id"] = r._id;
obj["name"] = r.name;
return obj;
});
response.send(result);
});
}
As per your above explanation about the schema design, i can assume your schema is as below :
Roles collection :
{
role_id :
name :
}
User collection :
{
user :
role_id : [ ]
....
}
you can use aggregation to achieve this :
db.users.aggregate([{$unwind: "$role_id"},
{$group: {_id: "$role_id", count : {"$sum":1}} },
]);

MongoDB and NodeJS get related data from 3 collections

i have a mongoDB query to get data with $group and $count.
This data contains the _id from other documents collection.
How can i get the other documents by its _id in NodeJS and MongoDB asyncrohnous?
db.orders.aggregate([
{$match: { 'works.TechnicianId': {$in:['53465f9d519c94680327965d','5383577a994be8b9a9e3f01e']},
'works.Date': {$gte: ISODate("2013-05-21T06:40:20.299Z"), $lt: ISODate("2016-05-21T06:40:20.299Z")}}},
{$unwind: "$works" },
{$group: {_id: "$works.TechnicianId",total:{$sum:'$works.price'},ordersId: { $push: "$_id" }}},
])
This is the result:
{
"result" : [
{
"_id" : "53465f9d519c94680327965d",
"total" : 198,
"ordersId" : [
ObjectId("537b5ea4c61b1d1743f4341f"),
ObjectId("537b4633021d75bd36863f29")
]
},
{
"_id" : "5383577a994be8b9a9e3f01e",
"total" : 22,
"ordersId" : [
ObjectId("537b5ea4c61b1d1743f4341f"),
ObjectId("537b4633021d75bd36863f29")
]
}
],
"ok" : 1
}
Now i need to get from orders collection the documents with id from ordersId, and from other collection the documents with _id from the result _id field.
I try with this:
var collection = db.collection('orders');
var result = [];
collection.aggregate([
{
$match: {
'works.TechnicianId': {
$in: ids
},
'works.Date': {
$gte: new Date(startDate),
$lt: new Date(endDate)
}
}
},
{
$unwind: "$works"
},
{
$group: {
_id: "$works.TechnicianId",
total: {
$sum: '$works.price'
},
orderId: {
$push: "$_id"
}
}
}
],
function (e, docs) {
if (e) {
error(e);
}
var usersCollection = db.collection('users');
_.each(docs, function (doc) {
usersCollection.findOne({_id: new ObjectID(doc._id)}, function (e, doc) {
doc.tech = doc;
});
doc.orders = [];
_.each(doc.orderId, function (queryOrder) {
collection.findOne({_id: new ObjectID(queryOrder._id)}, function (e, order) {
doc.orders.push(order);
});
});
success(docs);
});
});
But the success its called before all the _.eachs are finished..Any help, or idea?
Edit:
I try with Q promises, this is my code:
var usersCollection = db.collection('users');
var promises = [];
_.each(reports, function (report) {
var promise = usersCollection.findOne({_id: new ObjectID(report._id)}).then(
function (e, orderUserReported) {
if (e) {
error(e);
}
report.tech = orderUserReported;
_.each(orderUserReported.orderId, function (queryOrder) {
collection.findOne({_id: new ObjectID(queryOrder._id)}, function (e, order) {
report.orders.push(order);
});
});
});
promises.push(promise);
});
Q.allSettled(promises).then(success(reports));
and the error:
/Users/colymore/virteu/aa/server/node_modules/mongodb/lib/mongodb/connection/base.js:245
throw message;
^
TypeError: Cannot call method 'then' of undefined
Because of asynchronous execution you have to wait until results are returned. There are several options available:
async library https://github.com/caolan/async
promises https://github.com/kriskowal/q
Async is closer to your current code, you could use async.parallel https://github.com/caolan/async#parallel to wait untill you get data back
Update
Mongoose functions don't return Q promises, so you need to convert mongoose calls to promises by using something like Q.denodeify(User.findOne.bind(models.User))({ _id: userId}).then(...
For your case Q.denodeify(userCollection.findOne.bind(userCollection))({_id: new ObjectID(report._id)}).then(...
Short answer: Use promises. Look at Q.allSettled ( https://github.com/kriskowal/q )
Just run success asynchronously when all subtask are done.
Also using https://github.com/iolo/mongoose-q package may be helpful to not combine mongoose promises with Q ones if you want use mongoose in your mongo.

Resources