Optimize mongoDB query to get count of items from separate collection - node.js

I have two collections namely "tags" and "bookmarks".
Tags documents:
{
"taggedBookmarksCount": 2,
"taggedNotesCount": 0,
"_id": "627a80e6b12b0dc78b3a6d4b",
"name": "Article"
},
{
"taggedBookmarksCount": 0,
"taggedNotesCount": 0,
"_id": "62797885b479b5906ef6ed43",
"name": "Client"
},
Bookmark Documents:
{
"_id": "627a814db12b0dc78b3a6d54",
"bookmarkTags": [
{
"tagId": "627a814db12b0dc78b3a6d55",
"tag": "Article"
},
{
"tagId": "627a814db12b0dc78b3a6d56",
"tag": "to be read"
}
],
"bookmarkTitle": "Please sorrow of work",
}
Objective is to get the counts of bookmarks for all the tags in the "tags" collection.
Below is my current implementation, which returns the count of bookmarks for each tags.But this query takes around 3 sec to run (REST API response time) for 20 tags.
tags = await Tag.find(
{
userId: req.params.userId
},
{ _id: 1 }
);
tagIds = tags.map(tag => {
return tag._id.toString();
});
const tagCounts = await Promise.all(
tagIds.map(async tagId => {
const count = await Model.aggregate([
{
$match: {
bookmarkTags: {
$elemMatch: {
tagId: tagId
}
}
}
},
{
$group: {
_id: '_id',
count: {
$sum: 1
}
}
}
]);
return { tagId, count: count[0] ? count[0].count : 0 };
})
);
I am assuming its taking longer as I am mapping over all the tags, there are multiple round trips to database.Please suggest an approach to reduce the time of query execution.

You can do as below
db.bookmark.aggregate([
{
"$unwind": "$bookmarkTags" //Reshape tags
},
{
"$lookup": { //Do a join
"from": "tags",
"localField": "bookmarkTags.tagId",
"foreignField": "_id",
"as": "btags"
}
},
{
"$unwind": { //reshape the array elements
path: "$btags",
preserveNullAndEmptyArrays: true
}
},
{
"$group": { // Group tag wise bookmarks
"_id": "$bookmarkTags.tagId",
"docs": {
"$addToSet": "$btags"
}
}
},
{
"$project": { //Get counts, project what you want.
tag_id: "$_id",
"count": {
"$size": "$docs"
},
_id: 0
}
}
])
Playground
If you have given list of tag ids, then you can use it in match stage.
Updated playground
db.bookmark.aggregate([
{
"$unwind": "$bookmarkTags"
},
{
"$lookup": {
"from": "tags",
"localField": "bookmarkTags.tagId",
"foreignField": "_id",
"as": "btags"
}
},
{
"$unwind": {
path: "$btags",
preserveNullAndEmptyArrays: true
}
},
{
"$group": {
"_id": "$btags._id",
"docs": {
"$push": "$btags"
}
}
},
{
"$project": {
tag_id: "$_id",
"count": {
"$size": "$docs"
},
_id: 0
}
}
])

Related

Aggregate with multiple lookup and pipeline return only the last element

I try below code, it's working to lookup value from other collection. But why it only return the last element.
If I omitted the unwind function, It does return all result from the model, but the second lookup will not working as the first lookup return arrays.
My objective is to look up folder that include the model id which represented in templatefolders collection.
const result = await this.dashboardModel
.aggregate([{ $match: filter }])
.lookup({
from: 'templatefolders',
as: 'template',
let: { id: '$_id' },
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ['$dashboardId', '$$id'],
},
{
$eq: ['$deletedAt', null],
},
],
},
},
},
{
$project: {
_id: 1,
folderId: 1,
},
},
],
})
.unwind('template')
.lookup({
from: 'folders',
as: 'folder',
let: { folderId: '$template.folderId' },
pipeline: [
{
$match: {
$expr: {
$and: [
{
$eq: ['$_id', '$$folderId'],
},
{
$eq: ['$deletedAt', null],
},
],
},
},
},
{
$project: {
_id: 1,
name: 1,
},
},
],
})
.unwind('folder')
.exec();
return result;
Result
{
"data": [
{
...(parent field)
"template": {
"_id": "60ab22b03b39e40012b7cc4a",
"folderId": "60ab080b3b39e40012b7cc41"
},
"folder": {
"_id": "60ab080b3b39e40012b7cc41",
"name": "Folder 1"
}
}
],
"meta": {},
"success": true,
"message": "Succesfully get list"
}
I came from Front end background. I hope my question is not a silly one.
Thanks!
EDIT:
dashboard: [{
_id: dashboard1
}]
templatefolders: [{
dashboardId: dashboard1,
folderId: folder123
}]
folders: [{
_id: folder123
}]
You can use $lookup to join collections
$lookup to join two collections .Lookup doc
$unwind to deconstruct the array. Unwind doc
$group to reconstruct the array which we already deconstructed Group doc
Here is the code
db.dashboard.aggregate([
{
"$lookup": {
"from": "templatefolders",
"localField": "_id",
"foreignField": "dashboardId",
"as": "joinDashboard"
}
},
{
"$unwind": "$joinDashboard"
},
{
"$lookup": {
"from": "folders",
"localField": "joinDashboard.folderId",
"foreignField": "_id",
"as": "joinDashboard.joinFolder"
}
},
{
"$group": {
"_id": "$_id",
"joinDashboard": {
"$push": "$joinDashboard"
}
}
}
])
Working Mongo playground

Lookup when foreignField is in an Array

I want to lookup from an object to a collection where the foreignField key is embedded into an array of objects. I have:
collection "shirts"
{
"_id" : ObjectId("5a797ef0768d8418866eb0f6"),
"name" : "Supermanshirt",
"price" : 9.99,
"flavours" : [
{
"flavId" : ObjectId("5a797f8c768d8418866ebad3"),
"size" : "M",
"color": "white",
},
{
"flavId" : ObjectId("3a797f8c768d8418866eb0f7"),
"size" : "XL",
"color": "red",
},
]
}
collection "basket"
{
"_id" : ObjectId("5a797ef0333d8418866ebabc"),
"basketName" : "Default",
"items" : [
{
"dateAdded" : 1526996879787.0,
"itemFlavId" : ObjectId("5a797f8c768d8418866ebad3")
}
],
}
My Query:
basketSchema.aggregate([
{
$match: { $and: [{ _id }, { basketName }]},
},
{
$unwind: '$items',
},
{
$lookup:
{
from: 'shirts',
localField: 'items.itemFlavId',
foreignField: 'flavours.flavId',
as: 'ordered_shirts',
},
},
]).toArray();
my expected result:
[{
"_id" : ObjectId("5a797ef0333d8418866ebabc"),
"basketName" : "Default",
"items" : [
{
"dateAdded" : 1526996879787.0,
"itemFlavId" : ObjectId("5a797f8c768d8418866ebad3")
}
],
"ordered_shirts" : [
{
"_id" : ObjectId("5a797ef0768d8418866eb0f6"),
"name" : "Supermanshirt",
"price" : 9.99,
"flavours" : [
{
"flavId" : ObjectId("5a797f8c768d8418866ebad3"),
"size" : "M",
"color": "white",
}
]
}
],
}]
but instead my ordered_shirts array is empty.
How can I use a foreignField if this foreignField is embedded in an array at the other collection?
I am using MongoDB 3.6.4
As commented, it would appear that there is simply something up in your code where you are pointing at the wrong collection. The general case for this is to simply look at the example listing provided below and see what the differences are, since with the data you provide and the correct collection names then your expected result is in fact returned.
Of course where you need to take such a query "after" that initial $lookup stage is not a simple matter. From a structural standpoint, what you have is generally not a great idea since referring "joins" into items within an array means you are always returning data which is not necessarily "related".
There are some ways to combat that, and mostly there is the form of "non-correlated" $lookup introduced with MongoDB 3.6 which can aid in ensuring you are not returning "unnecessary" data in the "join".
I'm working here in the form of "merging" the "sku" detail with the "items" in the basket, so a first form would be:
Optimal MongoDB 3.6
// Store some vars like you have
let _id = ObjectId("5a797ef0333d8418866ebabc"),
basketName = "Default";
// Run non-correlated $lookup
let optimal = await Basket.aggregate([
{ "$match": { _id, basketName } },
{ "$lookup": {
"from": Shirt.collection.name,
"as": "items",
"let": { "items": "$items" },
"pipeline": [
{ "$match": {
"$expr": {
"$setIsSubset": ["$$items.itemflavId", "$flavours.flavId"]
}
}},
{ "$project": {
"_id": 0,
"items": {
"$map": {
"input": {
"$filter": {
"input": "$flavours",
"cond": { "$in": [ "$$this.flavId", "$$items.itemFlavId" ]}
}
},
"in": {
"$mergeObjects": [
{ "$arrayElemAt": [
"$$items",
{ "$indexOfArray": [
"$$items.itemFlavId", "$$this.flavId" ] }
]},
{ "name": "$name", "price": "$price" },
"$$this"
]
}
}
}
}},
{ "$unwind": "$items" },
{ "$replaceRoot": { "newRoot": "$items" } }
]
}}
])
Note that since you are using mongoose to hold details for the models we can use Shirt.collection.name here to read the property from that model with the actual collection name as needed for the $lookup. This helps avoid confusion within the code and also "hard-coding" something like the collection name when it's actually stored somewhere else. In this way should you change the code which registers the "model" in a way which altered the collection name, then this would always retrieve the correct name for use in the pipeline stage.
The main reason you use this form of $lookup with MongoDB 3.6 is because you want to use that "sub-pipeline" to manipulate the foreign collection results "before" they are returned and merged with the parent document. Since we are "merging" the results into the existing "items" array of the basket we use the same field name in argument to "as".
In this form of $lookup you typically still want "related" documents even though it gives you the control to do whatever you want. In this case we can compare the array content from "items" in the parent document which we set as a variable for the pipeline to use with the array under "flavours" in the foreign collection. A logical comparison for the two "sets" of values here where they "intersect" is using the $setIsSubset operator using the $expr so we can compare on a "logical operation".
The main work here is being done in the $project which is simply using $map on the array from the "flavours" array of the foreign document, processed with $filter in comparison to the "items" we passed into the pipeline and essentially re-written in order to "merge" the matched content.
The $filter reduces down the list for consideration to only those which match something present within the "items", and then we can use $indexOfArray and $arrayElemAt in order to extract the detail from the "items" and merge it with each remaining "flavours" entry which matches using the $mergeObjects operator. Noting here that we also take some "parent" detail from the "shirt" as the "name" and "price" fields which are common to the variations in size and color.
Since this is still an "array" within the matched document(s) to the join condition, in order to get a "flat list" of objects suitable for "merged" entries in the resulting "items" of the $lookup we simply apply $unwind, which within the context of matched items left only creates "little" overhead, and $replaceRoot in order to promote the content under that key to the top level.
The result is just the "merged" content listed in the "items" of the basket.
Sub-optimal MongoDB
The alternate approaches are really not that great since all involve returning other "flavours" which do not actually match the items in the basket. This basically involves "post-filtering" the results obtained from the $lookup as opposed to "pre-filtering" which the process above does.
So the next case here would be using methods to manipulate the returned array in order to remove the items which don't actually match:
// Using legacy $lookup
let alternate = await Basket.aggregate([
{ "$match": { _id, basketName } },
{ "$lookup": {
"from": Shirt.collection.name,
"localField": "items.itemFlavId",
"foreignField": "flavours.flavId",
"as": "ordered_items"
}},
{ "$addFields": {
"items": {
"$let": {
"vars": {
"ordered_items": {
"$reduce": {
"input": {
"$map": {
"input": "$ordered_items",
"as": "o",
"in": {
"$map": {
"input": {
"$filter": {
"input": "$$o.flavours",
"cond": {
"$in": ["$$this.flavId", "$items.itemFlavId"]
}
}
},
"as": "f",
"in": {
"$mergeObjects": [
{ "name": "$$o.name", "price": "$$o.price" },
"$$f"
]
}
}
}
}
},
"initialValue": [],
"in": { "$concatArrays": ["$$value", "$$this"] }
}
}
},
"in": {
"$map": {
"input": "$items",
"in": {
"$mergeObjects": [
"$$this",
{ "$arrayElemAt": [
"$$ordered_items",
{ "$indexOfArray": [
"$$ordered_items.flavId", "$$this.itemFlavId"
]}
]}
]
}
}
}
}
},
"ordered_items": "$$REMOVE"
}}
]);
Here I'm still using some MongoDB 3.6 features, but these are not a "requirement" of the logic involved. The main constraint in this approach is actually the $reduce which requires MongoDB 3.4 or greater.
Using the same "legacy" form of $lookup as you were attempting, we still get the desired results as you display but that of course contains information in the "flavours" that does not match the "items" in the basket. In much the same way as shown in the previous listing we can apply $filter here to remove the items which don't match. The same process here uses that $filter output as the input for $map, which again is doing the same "merge" process as before.
Where the $reduce comes in is because the resulting processing where there is an "array" target from $lookup with documents that themselves contain an "array" of "flavours" is that these arrays need to be "merged" into a single array for further processing. The $reduce simply uses the processed output and performs a $concatArrays on each of the "inner" arrays returned to make these results singular. We already "merged" the content, so this becomes the new "merged" "items".
Older Still $unwind
And of course the final way to present ( even though there are other combinations ) is using $unwind on the arrays and using $group to put it back together:
let old = await Basket.aggregate([
{ "$match": { _id, basketName } },
{ "$unwind": "$items" },
{ "$lookup": {
"from": Shirt.collection.name,
"localField": "items.itemFlavId",
"foreignField": "flavours.flavId",
"as": "ordered_items"
}},
{ "$unwind": "$ordered_items" },
{ "$unwind": "$ordered_items.flavours" },
{ "$redact": {
"$cond": {
"if": {
"$eq": [
"$items.itemFlavId",
"$ordered_items.flavours.flavId"
]
},
"then": "$$KEEP",
"else": "$$PRUNE"
}
}},
{ "$group": {
"_id": "$_id",
"basketName": { "$first": "$basketName" },
"items": {
"$push": {
"dateAdded": "$items.dateAdded",
"itemFlavId": "$items.itemFlavId",
"name": "$ordered_items.name",
"price": "$ordered_items.price",
"flavId": "$ordered_items.flavours.flavId",
"size": "$ordered_items.flavours.size",
"color": "$ordered_items.flavours.color"
}
}
}}
]);
Most of this should be pretty self explanatory as $unwind is simply a tool to "flatten" array content into singular document entries. In order to just get the results we want we can use $redact to compare the two fields. Using MongoDB 3.6 you "could" use $expr within a $match here:
{ "$match": {
"$expr": {
"$eq": [
"$items.itemFlavId",
"$ordered_items.flavours.flavId"
]
}
}}
But when it comes down to it, if you have MongoDB 3.6 with it's other features then $unwind is the wrong thing to do here due to all the overhead it will actually add.
So all that really happens is you $lookup then "flatten" the documents and finally $group all related detail together using $push to recreate the "items" in the basket. It "looks simple" and is probably the most easy form to understand, however "simplicity" does not equal "performance" and this would be pretty brutal to use in a real world use case.
Summary
That should cover the explanation of the things you need to do when working with "joins" that are going to compare items within arrays. This probably should lead you on the path of realizing this is not really a great idea and it would be far better to keep your "skus" listed "separately" rather than listing them all related under a single "item".
It also should in part be a lesson that "joins" in general are not a great idea with MongoDB. You really only should define such relations where they are "absolutely necessary". In such a case of "details for items in a basket", then contrary to traditional RDBMS patterns it would actually be far better in terms of performance to simply "embed" that detail from the start. In that way you don't need complicated join conditions just to get a result, which might have saved "a few bytes" in storage but is taking a lot more time than what should have been a simple request for the basket with all the detail already "embedded". That really should be the primary reason why you are using something like MongoDB in the first place.
So if you have to do it, then really you should be sticking with the first form since where you have the available features to use then use them best to their advantage. Whilst other approaches may seem easier, it won't help the application performance, and of course best performance would be embedding to begin with.
A full listing follows for demonstration of the above discussed methods and for basic comparison to prove that the provided data does in fact "join" as long as the other parts of the application set-up are working as they should be. So a model on "how it should be done" in addition to demonstrating the full concepts.
const { Schema, Types: { ObjectId } } = mongoose = require('mongoose');
const uri = 'mongodb://localhost/basket';
mongoose.Promise = global.Promise;
mongoose.set('debug', true);
const basketItemSchema = new Schema({
dateAdded: { type: Number, default: Date.now() },
itemFlavId: { type: Schema.Types.ObjectId }
},{ _id: false });
const basketSchema = new Schema({
basketName: String,
items: [basketItemSchema]
});
const flavourSchema = new Schema({
flavId: { type: Schema.Types.ObjectId },
size: String,
color: String
},{ _id: false });
const shirtSchema = new Schema({
name: String,
price: Number,
flavours: [flavourSchema]
});
const Basket = mongoose.model('Basket', basketSchema);
const Shirt = mongoose.model('Shirt', shirtSchema);
const log = data => console.log(JSON.stringify(data, undefined, 2));
(async function() {
try {
const conn = await mongoose.connect(uri);
// clean data
await Promise.all(Object.entries(conn.models).map(([k,m]) => m.remove()));
// set up data for test
await Basket.create({
_id: ObjectId("5a797ef0333d8418866ebabc"),
basketName: "Default",
items: [
{
dateAdded: 1526996879787.0,
itemFlavId: ObjectId("5a797f8c768d8418866ebad3")
}
]
});
await Shirt.create({
_id: ObjectId("5a797ef0768d8418866eb0f6"),
name: "Supermanshirt",
price: 9.99,
flavours: [
{
flavId: ObjectId("5a797f8c768d8418866ebad3"),
size: "M",
color: "white"
},
{
flavId: ObjectId("3a797f8c768d8418866eb0f7"),
size: "XL",
color: "red"
}
]
});
// Store some vars like you have
let _id = ObjectId("5a797ef0333d8418866ebabc"),
basketName = "Default";
// Run non-correlated $lookup
let optimal = await Basket.aggregate([
{ "$match": { _id, basketName } },
{ "$lookup": {
"from": Shirt.collection.name,
"as": "items",
"let": { "items": "$items" },
"pipeline": [
{ "$match": {
"$expr": {
"$setIsSubset": ["$$items.itemflavId", "$flavours.flavId"]
}
}},
{ "$project": {
"_id": 0,
"items": {
"$map": {
"input": {
"$filter": {
"input": "$flavours",
"cond": { "$in": [ "$$this.flavId", "$$items.itemFlavId" ]}
}
},
"in": {
"$mergeObjects": [
{ "$arrayElemAt": [
"$$items",
{ "$indexOfArray": [
"$$items.itemFlavId", "$$this.flavId" ] }
]},
{ "name": "$name", "price": "$price" },
"$$this"
]
}
}
}
}},
{ "$unwind": "$items" },
{ "$replaceRoot": { "newRoot": "$items" } }
]
}}
])
log(optimal);
// Using legacy $lookup
let alternate = await Basket.aggregate([
{ "$match": { _id, basketName } },
{ "$lookup": {
"from": Shirt.collection.name,
"localField": "items.itemFlavId",
"foreignField": "flavours.flavId",
"as": "ordered_items"
}},
{ "$addFields": {
"items": {
"$let": {
"vars": {
"ordered_items": {
"$reduce": {
"input": {
"$map": {
"input": "$ordered_items",
"as": "o",
"in": {
"$map": {
"input": {
"$filter": {
"input": "$$o.flavours",
"cond": {
"$in": ["$$this.flavId", "$items.itemFlavId"]
}
}
},
"as": "f",
"in": {
"$mergeObjects": [
{ "name": "$$o.name", "price": "$$o.price" },
"$$f"
]
}
}
}
}
},
"initialValue": [],
"in": { "$concatArrays": ["$$value", "$$this"] }
}
}
},
"in": {
"$map": {
"input": "$items",
"in": {
"$mergeObjects": [
"$$this",
{ "$arrayElemAt": [
"$$ordered_items",
{ "$indexOfArray": [
"$$ordered_items.flavId", "$$this.itemFlavId"
]}
]}
]
}
}
}
}
},
"ordered_items": "$$REMOVE"
}}
]);
log(alternate);
// Or really old style
let old = await Basket.aggregate([
{ "$match": { _id, basketName } },
{ "$unwind": "$items" },
{ "$lookup": {
"from": Shirt.collection.name,
"localField": "items.itemFlavId",
"foreignField": "flavours.flavId",
"as": "ordered_items"
}},
{ "$unwind": "$ordered_items" },
{ "$unwind": "$ordered_items.flavours" },
{ "$redact": {
"$cond": {
"if": {
"$eq": [
"$items.itemFlavId",
"$ordered_items.flavours.flavId"
]
},
"then": "$$KEEP",
"else": "$$PRUNE"
}
}},
{ "$group": {
"_id": "$_id",
"basketName": { "$first": "$basketName" },
"items": {
"$push": {
"dateAdded": "$items.dateAdded",
"itemFlavId": "$items.itemFlavId",
"name": "$ordered_items.name",
"price": "$ordered_items.price",
"flavId": "$ordered_items.flavours.flavId",
"size": "$ordered_items.flavours.size",
"color": "$ordered_items.flavours.color"
}
}
}}
]);
log(old);
} catch(e) {
console.error(e)
} finally {
process.exit()
}
})()
And sample output as:
Mongoose: baskets.remove({}, {})
Mongoose: shirts.remove({}, {})
Mongoose: baskets.insertOne({ _id: ObjectId("5a797ef0333d8418866ebabc"), basketName: 'Default', items: [ { dateAdded: 1526996879787, itemFlavId: ObjectId("5a797f8c768d8418866ebad3") } ], __v: 0 })
Mongoose: shirts.insertOne({ _id: ObjectId("5a797ef0768d8418866eb0f6"), name: 'Supermanshirt', price: 9.99, flavours: [ { flavId: ObjectId("5a797f8c768d8418866ebad3"), size: 'M', color: 'white' }, { flavId: ObjectId("3a797f8c768d8418866eb0f7"), size: 'XL', color: 'red' } ], __v: 0 })
Mongoose: baskets.aggregate([ { '$match': { _id: 5a797ef0333d8418866ebabc, basketName: 'Default' } }, { '$lookup': { from: 'shirts', as: 'items', let: { items: '$items' }, pipeline: [ { '$match': { '$expr': { '$setIsSubset': [ '$$items.itemflavId', '$flavours.flavId' ] } } }, { '$project': { _id: 0, items: { '$map': { input: { '$filter': { input: '$flavours', cond: { '$in': [Array] } } }, in: { '$mergeObjects': [ { '$arrayElemAt': [Array] }, { name: '$name', price: '$price' }, '$$this' ] } } } } }, { '$unwind': '$items' }, { '$replaceRoot': { newRoot: '$items' } } ] } } ], {})
[
{
"_id": "5a797ef0333d8418866ebabc",
"basketName": "Default",
"items": [
{
"dateAdded": 1526996879787,
"itemFlavId": "5a797f8c768d8418866ebad3",
"name": "Supermanshirt",
"price": 9.99,
"flavId": "5a797f8c768d8418866ebad3",
"size": "M",
"color": "white"
}
],
"__v": 0
}
]
Mongoose: baskets.aggregate([ { '$match': { _id: 5a797ef0333d8418866ebabc, basketName: 'Default' } }, { '$lookup': { from: 'shirts', localField: 'items.itemFlavId', foreignField: 'flavours.flavId', as: 'ordered_items' } }, { '$addFields': { items: { '$let': { vars: { ordered_items: { '$reduce': { input: { '$map': { input: '$ordered_items', as: 'o', in: { '$map': [Object] } } }, initialValue: [], in: { '$concatArrays': [ '$$value', '$$this' ] } } } }, in: { '$map': { input: '$items', in: { '$mergeObjects': [ '$$this', { '$arrayElemAt': [ '$$ordered_items', [Object] ] } ] } } } } }, ordered_items: '$$REMOVE' } } ], {})
[
{
"_id": "5a797ef0333d8418866ebabc",
"basketName": "Default",
"items": [
{
"dateAdded": 1526996879787,
"itemFlavId": "5a797f8c768d8418866ebad3",
"name": "Supermanshirt",
"price": 9.99,
"flavId": "5a797f8c768d8418866ebad3",
"size": "M",
"color": "white"
}
],
"__v": 0
}
]
Mongoose: baskets.aggregate([ { '$match': { _id: 5a797ef0333d8418866ebabc, basketName: 'Default' } }, { '$unwind': '$items' }, { '$lookup': { from: 'shirts', localField: 'items.itemFlavId', foreignField: 'flavours.flavId', as: 'ordered_items' } }, { '$unwind': '$ordered_items' }, { '$unwind': '$ordered_items.flavours' }, { '$redact': { '$cond': { if: { '$eq': [ '$items.itemFlavId', '$ordered_items.flavours.flavId' ] }, then: '$$KEEP', else: '$$PRUNE' } } }, { '$group': { _id: '$_id', basketName: { '$first': '$basketName' }, items: { '$push': { dateAdded: '$items.dateAdded', itemFlavId: '$items.itemFlavId', name: '$ordered_items.name', price: '$ordered_items.price', flavId: '$ordered_items.flavours.flavId', size: '$ordered_items.flavours.size', color: '$ordered_items.flavours.color' } } } } ], {})
[
{
"_id": "5a797ef0333d8418866ebabc",
"basketName": "Default",
"items": [
{
"dateAdded": 1526996879787,
"itemFlavId": "5a797f8c768d8418866ebad3",
"name": "Supermanshirt",
"price": 9.99,
"flavId": "5a797f8c768d8418866ebad3",
"size": "M",
"color": "white"
}
]
}
]

Mongoose Aggregate with Lookup

I have a simple two collections like below :
assignments:
[
{
"_id": "593eff62630a1c35781fa325",
"topic_id": 301,
"user_id": "59385ef6d2d80c00d9bdef97"
},
{
"_id": "593eff62630a1c35781fa326",
"topic_id": 301,
"user_id": "59385ef6d2d80c00d9bdef97"
}
]
and users collection:
[
{
"_id": "59385ef6d2d80c00d9bdef97",
"name": "XX"
},
{
"_id": "59385b547e8918009444a3ac",
"name": "YY"
}
]
and my intent is, an aggregate query by user_id on assignment collection, and also I would like to include user.name in that group collection. I tried below:
Assignment.aggregate([{
$match: {
"topic_id": "301"
}
},
{
$group: {
_id: "$user_id",
count: {
$sum: 1
}
}
},
{
$lookup: {
"from": "kullanicilar",
"localField": "user_id",
"foreignField": "_id",
"as": "user"
}
},
{
$project: {
"user": "$user",
"count": "$count",
"_id": "$_id"
}
},
But the problem is that user array is always blank.
[ { _id: '59385ef6d2d80c00d9bdef97', count: 1000, user: [] } ]
I want something like :
[ { _id: '59385ef6d2d80c00d9bdef97', count: 1000, user: [_id:"59385ef6d2d80c00d9bdef97",name:"XX"] } ]

How to Populate and Aggregate

i want to aggregate the ratings so i could get the total of feedbacks. But the thing is, it's referenced. Here's my schema
User
username: String,
fullname: String,
email: {
type: String,
lowercase: true,
unique: true
},
address: String,
password: String,
feedback: [{
type: mongoose.Schema.Types.ObjectId,
ref: 'Feedback'
}]
Feedback
var FeedbackSchema = new mongoose.Schema({
postname: String,
user: {
type: mongoose.Schema.Types.ObjectId,
ref: 'User'
},
message: String,
feedbacktype: String,
thumbsup: Boolean,
rating: {
communication: Number,
timeliness: Number,
delivery: Number
}
});
So what i want to achieve is, i will find the User by id then populate the feedbacks field, then i will aggregate the ratings on the feedback field so i would get a total of number of ratings for communication, for delivery and for timeliness. (The ratings are 1-5 stars)
Do you know how to aggregate and populate? thank you
**update
So i've run the aggregation to the user schema, now im getting 0 results from all ratings
User.aggregate([
{ "$match": { "_id": ObjectId('593150f6ac4d9b0410d2aac0') } },
{ "$lookup": {
"from": "feedbacks",
"localField": "feedback",
"foreignField": "_id",
"as": "feedback"
}},
{ "$project": {
"username": 1,
"fullname": 1,
"email": 1,
"password": 1,
"rating": {
"communication": { "$sum": "$feedback.rating.communication" },
"timeliness": { "$sum": "$feedback.rating.timeliness" },
"delivery": { "$sum": "$feedback.rating.delivery" }
}
}}
]).exec(function(err, a){
console.log(a)
})
result rating: { communication: 0, timeliness: 0, delivery: 0 } } ]
also tried it with other users, all of them 0 result rating
Simple Listing to Follow
var async = require('async'),
mongoose = require('mongoose'),
Schema = mongoose.Schema;
mongoose.connect('mongodb://localhost/rating');
var userSchema = new Schema({
username: String,
feedback: [{ type: Schema.Types.ObjectId, ref: 'Feedback' }]
});
var feedbackSchema = new Schema({
rating: {
communication: Number,
timeliness: Number,
delivery: Number
}
});
var User = mongoose.model('User', userSchema);
var Feedback = mongoose.model('Feedback', feedbackSchema);
async.series(
[
(callback) => {
async.each([User,Feedback],(model,callback) => {
model.remove({},callback);
},callback);
},
(callback) => {
async.waterfall(
[
(callback) => {
async.map(
[
{ "rating": {
"communication": 1, "timeliness": 2, "delivery": 3
}},
{ "rating": {
"communication": 2, "timeliness": 3, "delivery": 4
}}
],
(item,callback) => {
Feedback.create(item,callback)
},
callback
);
},
(feedback, callback) => {
User.create({ "username": "Bill", "feedback": feedback },callback);
},
(user, callback) => {
User.aggregate([
{ "$match": { "_id": user._id } },
{ "$lookup": {
"from": "feedbacks",
"localField": "feedback",
"foreignField": "_id",
"as": "feedback"
}},
{ "$project": {
"username": 1,
"rating": {
"communication": { "$sum": "$feedback.rating.communication" },
"timeliness": { "$sum": "$feedback.rating.timeliness" },
"delivery": { "$sum": "$feedback.rating.delivery" }
}
}}
],(err,results) => {
console.log(JSON.stringify(results, undefined, 2));
callback(err);
});
}
],
callback
)
}
],
(err) => {
if (err) throw err;
mongoose.disconnect();
}
);
This will create two collections as User
{
"_id" : ObjectId("593548455198ab3c09cf736b"),
"username" : "Bill",
"feedback" : [
ObjectId("593548455198ab3c09cf7369"),
ObjectId("593548455198ab3c09cf736a")
],
"__v" : 0
}
And feedbacks:
{
"_id" : ObjectId("593548455198ab3c09cf7369"),
"rating" : {
"communication" : 1,
"timeliness" : 2,
"delivery" : 3
},
"__v" : 0
}
{
"_id" : ObjectId("593548455198ab3c09cf736a"),
"rating" : {
"communication" : 2,
"timeliness" : 3,
"delivery" : 4
},
"__v" : 0
}
Program Output Shows the aggregation:
[
{
"_id": "5935494a159c633c1b34807b",
"username": "Bill",
"rating": {
"communication": 3,
"timeliness": 5,
"delivery": 7
}
}
]
Also package.json if the two dependencies are not clear enough:
{
"name": "ratings",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"async": "^2.4.1",
"mongoose": "^4.10.4"
}
}
Original Answer
Personally I would work this from the "Feedback" since you have the user already recorded there, and it is actually the way this scales better.
Instead of using population, we can instead use $lookup with a MongoDB server version of at least 3.2:
Feedback.aggregate([
{ "$match": { "user": userId } },
{ "$group": {
"_id": "$user",
"communication": { "$sum": "$rating.communication" },
"timeliness": { "$sum": "$rating.timeliness" },
"delivery": { "$sum": "$rating.delivery" }
}},
{ "$lookup": {
"from": "users",
"localField": "_id",
"foreignField": "_id",
"as": "user"
}},
{ "$unwind": "$user" }
])
If you do not have a server version that supports $lookupthen you can still "manually join" the User details with something like this:
Feedback.aggregate([
{ "$match": { "user": userId } },
{ "$group": {
"_id": "$user",
"communication": { "$sum": "$rating.communication" },
"timeliness": { "$sum": "$rating.timeliness" },
"delivery": { "$sum": "$rating.delivery" }
}}
],function(err, results) {
result = results[0];
User.findById(userId).lean().exec(function(err, user) {
result.user = user; // swap the _id for the Object
// Then output result
});
})
Which is basically what .populate() does, but we are doing it manually and efficiently for the result returned.
You can work the other way around from the User model, but it's likely more efficient to simply work this way around.
User.aggregate([
{ "$match": { "_id": userid } },
{ "$lookup": {
"from": "feedbacks",
"localField": "feedback",
"foreignField": "_id",
"as": "feedback"
}},
{ "$project": {
"username": 1,
"fullname": 1,
"email": 1,
"password": 1,
"rating": {
"communication": { "$sum": "$feedback.rating.communication" },
"timeliness": { "$sum": "$feedback.rating.timeliness" },
"delivery": { "$sum": "$feedback.rating.delivery" }
}
}}
])

Mongoose: how to use aggregate and find together

How can I use aggregate and find together in Mongoose?
i.e I have the following schema:
const schema = new Mongoose.Schema({
created: { type: Date, default: Date.now() },
name: { type: String, default: 'development' }
followers: [{ type: Mongoose.Schema.ObjectId, ref: 'Users'}]
...
})
export default Mongoose.model('Locations', schema)
How can I query the users with only the fields name and followers_count.
followers_count: the length of followers .
There, I know we can use select to get only the field name.
How can we get the count of followers?
For MongoDB 3.6 and greater, use the $expr operator which allows the use of aggregation expressions within the query language:
var followers_count = 30;
db.locations.find({
"$expr": {
"$and": [
{ "$eq": ["$name", "development"] },
{ "$gte": [{ "$size": "$followers" }, followers_count ]}
]
}
});
For non-compatible versions, you can use both the $match and $redact pipelines to query your collection. For example, if you want to query the locations collection where the name is 'development' and followers_count is greater than 30, run the following aggregate operation:
const followers_count = 30;
Locations.aggregate([
{ "$match": { "name": "development" } },
{
"$redact": {
"$cond": [
{ "$gte": [ { "$size": "$followers" }, followers_count ] },
"$$KEEP",
"$$PRUNE"
]
}
}
]).exec((err, locations) => {
if (err) throw err;
console.log(locations);
})
or within a single pipeline as
Locations.aggregate([
{
"$redact": {
"$cond": [
{
"$and": [
{ "$eq": ["$name", "development"] },
{ "$gte": [ { "$size": "$followers" }, followers_count ] }
]
},
"$$KEEP",
"$$PRUNE"
]
}
}
]).exec((err, locations) => {
if (err) throw err;
console.log(locations);
})
The above will return the locations with just the _id references from the users. To return the users documents as means to "populate" the followers array, you can then append the $lookup pipeline.
If the underlying Mongo server version is 3.4 and newer, you can run the pipeline as
let followers_count = 30;
Locations.aggregate([
{ "$match": { "name": "development" } },
{
"$redact": {
"$cond": [
{ "$gte": [ { "$size": "$followers" }, followers_count ] },
"$$KEEP",
"$$PRUNE"
]
}
},
{
"$lookup": {
"from": "users",
"localField": "followers",
"foreignField": "_id",
"as": "followers"
}
}
]).exec((err, locations) => {
if (err) throw err;
console.log(locations);
})
else you would need to $unwind the followers array before applying $lookup and then regroup with $group pipeline after that:
let followers_count = 30;
Locations.aggregate([
{ "$match": { "name": "development" } },
{
"$redact": {
"$cond": [
{ "$gte": [ { "$size": "$followers" }, followers_count ] },
"$$KEEP",
"$$PRUNE"
]
}
},
{ "$unwind": "$followers" },
{
"$lookup": {
"from": "users",
"localField": "followers",
"foreignField": "_id",
"as": "follower"
}
},
{ "$unwind": "$follower" },
{
"$group": {
"_id": "$_id",
"created": { "$first": "$created" },
"name": { "$first": "$name" },
"followers": { "$push": "$follower" }
}
}
]).exec((err, locations) => {
if (err) throw err;
console.log(locations);
})
You can use as the following:
db.locations.aggregate([
{$match:{"your find query"}},
{$project:{"your desired fields"}}
])
In the match you can do stuff like:
{{$match:{name:"whatever"}}
In the project, you can select the fields you want using numbers 0 or 1 like:
{$project:{_id:1,created:0,name:1}}
Which 0 means, do not put and 1 means put.

Resources