Related
I would like to "create" new documents, to keep only the contents of the activities.
I've looked at $project and $unwide, but I didn't find any appropriate result.
Here is my MongoDB query:
const activities = await friendModel.aggregate([
{
$match: {
$or: [
{
ReceiverId: ActorId,
Status: 1,
},
{
RequesterId: ActorId,
Status: 1,
},
],
},
},
{
$set: {
fieldResult: {
$cond: {
if: {
$eq: [ "$ReceiverId", ActorId ],
},
then: "$RequesterId",
else: "$ReceiverId",
},
},
},
},
{
$lookup: {
from: "activities",
localField: "fieldResult",
foreignField: "ActorId",
as: "activity",
},
},
{
$unwind: {
path: "$activity",
preserveNullAndEmptyArrays: true
}
},
{ $sort: { "activity._Date": -1 } },
{ $skip: request.pageindex * 3 },
{ $limit: 3 },
]);
Here is what I would like to do: https://sourceb.in/7u0tirIstt
Here is what I get for the moment: https://sourceb.in/Ped2xY5Ml8
Thank you in advance for your help!
You should be able to add $project as follows:
...
{
$project: {
"_id":"$_id",
"ActivityId": "$activity._id",
"ActorId": "$activity.ActorId",
"Type": "$activity.Type",
"_Date": "$activity._Date",
"MovieId": "$activity.MovieId",
"FriendId": "$activity.FriendId",
"ContestId": "$activity.ContestId",
"LookId": "$activity.LookId",
"__v": "$__v"
},
},
...
You can access values with $, and map nested values by accessing them directly
Also would recommend you doing a limit before you do lookup to speed it up a bit
This is Code in node js
const result = await OrderDB.aggregate([
{ $match: { _id: mongoose.Types.ObjectId(id) } },
{
$lookup: {
from: 'products',
localField: 'product',
foreignField: '_id',
as: 'productDetail',
},
},
{
$project: {
productDetail: {
name: 1,
price: 1,
productImage: 1,
},
},
},
])
This is the response of code
{
"message": "Get Order Successfully",
"result": [
{
"_id": "5ff47348db5f5917f81871aa",
"productDetail": [
{
"name": "Camera",
"productImage": [
{
"_id": "5fe9b8a26720f728b814e246",
"img": "uploads\\product\\7Rq1v-app-7.jpg"
},
{
"_id": "5fe9b8a26720f728b814e247",
"img": "uploads\\product\\FRuVb-app-8.jpg"
}
],
"price": 550
}
]
}
]
}
I want to display only one productImage from the response using nodejs and mongoose
This is using in aggregate projection
I was use $arrayElemAt but it is don't work
I also use $first but it is don't work
so projection method I use to display only one *productImage*
Data looks like multi level nested.
You have array of results, each result contains array of productDetails
play
You need to unwind the data to get the first productImage
db.collection.aggregate([
{
"$unwind": "$result"
},
{
"$unwind": "$result.productDetail"
},
{
$project: {
pImage: {
"$first": "$result.productDetail.productImage"
}
}
}
])
With the above response
db.collection.aggregate([
{
"$project": {
productDetails: {
$map: {
input: "$productDetail",
in: {
"$mergeObjects": [
"$$this",
{
productImage: {
"$arrayElemAt": [
"$$this.productImage",
0
]
}
}
]
}
}
}
}
}
])
Working Mongo playground
This question is a follow up to a previous question for which I have accepted an answer already. I have an aggregate query that returns the results of a deeply nested array of subdocuments based on a date range. The query returns the correct results within the specified date range, however it also returns an empty array for the results that do not match the query.
Technologies: MongoDB 3.6, Mongoose 5.5, NodeJS 12
Question 1:
Is there any way to remove the results that don't match the query?
Question 2:
Is there any way to 'populate' the Person db reference in the results? For example to get the Person Display Name I usually use 'populate' such as find().populate({ path: 'Person', select: 'DisplayName'})
Records schema
let RecordsSchema = new Schema({
RecordID: {
type: Number,
index: true
},
RecordType: {
type: String
},
Status: {
type: String
},
// ItemReport array of subdocuments
ItemReport: [ItemReportSchema],
}, {
collection: 'records',
selectPopulatedPaths: false
});
let ItemReportSchema = new Schema({
// ObjectId reference
ReportBy: {
type: Schema.Types.ObjectId,
ref: 'people'
},
ReportDate: {
type: Date,
required: true
},
WorkDoneBy: [{
Person: {
type: Schema.Types.ObjectId,
ref: 'people'
},
CompletedHours: {
type: Number,
required: true
},
DateCompleted: {
type: Date
}
}],
});
Query
Works but also returns empty results and also need to populate the Display Name property of the Person db reference
db.records.aggregate([
{
"$project": {
"ItemReport": {
$map: {
input: "$ItemReport",
as: "ir",
in: {
WorkDoneBy: {
$filter: {
input: "$$ir.WorkDoneBy",
as: "value",
cond: {
"$and": [
{ "$ne": [ "$$value.DateCompleted", null ] },
{ "$gt": [ "$$value.DateCompleted", new Date("2017-01-01T12:00:00.000Z") ] },
{ "$lt": [ "$$value.DateCompleted", new Date("2018-12-31T12:00:00.000Z") ] }
]
}
}
}
}
}
}
}
}
])
Actual Results
{
"_id": "5dcb6406e63830b7aa5427ca",
"ItemReport": [
{
"WorkDoneBy": [
{
"_id": "5dcb6406e63830b7aa53d8ea",
"PersonID": 111,
"ReportID": 8855,
"CompletedHours": 3,
"DateCompleted": "2017-01-20T05:00:00.000Z",
"Person": "5dcb6409e63830b7aa54fdba"
}
]
}
]
},
{
"_id": "5dcb6406e63830b7aa5427f1",
"ItemReport": [
{
"WorkDoneBy": [
{
"_id": "5dcb6406e63830b7aa53dcdc",
"PersonID": 4,
"ReportID": 9673,
"CompletedHours": 17,
"DateCompleted": "2017-05-18T04:00:00.000Z",
"Person": "5dcb6409e63830b7aa54fd69"
},
{
"_id": "5dcb6406e63830b7aa53dcdd",
"PersonID": 320,
"ReportID": 9673,
"CompletedHours": 3,
"DateCompleted": "2017-05-18T04:00:00.000Z",
"Person": "5dcb6409e63830b7aa54fe88"
}
]
}
]
},
{
"_id": "5dcb6406e63830b7aa5427f2",
"ItemReport": [
{
"WorkDoneBy": []
}
]
},
{
"_id": "5dcb6406e63830b7aa5427f3",
"ItemReport": [
{
"WorkDoneBy": []
}
]
},
{
"_id": "5dcb6406e63830b7aa5427f4",
"ItemReport": [
{
"WorkDoneBy": []
}
]
},
{
"_id": "5dcb6406e63830b7aa5427f5",
"ItemReport": [
{
"WorkDoneBy": []
}
]
},
Desired results
Note the results with an empty "WorkDoneBy" array are removed (question 1), and the "Person" display name is populated (question 2).
{
"_id": "5dcb6406e63830b7aa5427f1",
"ItemReport": [
{
"WorkDoneBy": [
{
"_id": "5dcb6406e63830b7aa53dcdc",
"CompletedHours": 17,
"DateCompleted": "2017-05-18T04:00:00.000Z",
"Person": {
_id: "5dcb6409e63830b7aa54fe88",
DisplayName: "Joe Jones"
}
},
{
"_id": "5dcb6406e63830b7aa53dcdd",
"CompletedHours": 3,
"DateCompleted": "2017-05-18T04:00:00.000Z",
"Person": {
_id: "5dcb6409e63830b7aa54fe88",
DisplayName: "Alice Smith"
}
}
]
}
]
},
First question is relatively easy to answer and there are multiple ways to do that. I would prefer using $anyElementTrue along with $map as those operators are pretty self-explanatory.
{
"$match": {
$expr: { $anyElementTrue: { $map: { input: "$ItemReport", in: { $gt: [ { $size: "$$this.WorkDoneBy" }, 0 ] } } } }
}
}
MongoPlayground
Second part is a bit more complicated but still possible. Instead of populate you need to run $lookup to bring the data from other collection. The problem is that your Person values are deeply nested so you need to prepare a list of id values before using $reduce and $setUnion. Once you get the data you need to merge your nested objects with people entities using $map and $mergeObjects.
{
$addFields: {
people: {
$reduce: {
input: "$ItemReport",
initialValue: [],
in: { $setUnion: [ "$$value", "$$this.WorkDoneBy.Person" ] }
}
}
}
},
{
$lookup: {
from: "people",
localField: "peopleIds",
foreignField: "_id",
as: "people"
}
},
{
$project: {
_id: 1,
ItemReport: {
$map: {
input: "$ItemReport",
as: "ir",
in: {
WorkDoneBy: {
$map: {
input: "$$ir.WorkDoneBy",
as: "wdb",
in: {
$mergeObjects: [
"$$wdb",
{
Person: { $arrayElemAt: [{ $filter: { input: "$people", cond: { $eq: [ "$$this._id", "$$wdb.Person" ] } } } , 0] }
}
]
}
}
}
}
}
}
}
}
Complete Solution
I'm struggling with a problem of how to get the number of distinct field values in array returned as result of $lookup aggregation step in MongoDB using Mongoose. By the number of distinct field values I mean the number of rows with unique value on certain field.
Parent document has this structure:
{ _id: 678, name: "abc" }
Child document has this structure:
{ _id: 1009, fieldA: 123, x: { id: 678, name: "abc" } }
$lookup step is defined as follow:
{
from "children",
localField: "_id"
foreignField: "x.id"
as: "xyz"
}
Let's assume that I get this array as a result of $lookup aggregation step for a parent with _id equal to: 678
xyz: [
{ _id: 1009, fieldA: 123, x: { id: 678, name: "abc" } },
{ _id: 1010, fieldA: 3435, x: { id: 678, name: "abc" } },
{ _id: 1011, fieldA: 123, x: { id: 678, name: "abc" } }
]
I want to know how many distinct fieldA values are in this array. In this example it would be 2.
Of coure the step should be in aggregation flow, ater $lookup step and before (inside?) $project step. As a side note I must to add that I also need total number of elements in array xyz as another value ($size operator in $project step).
So given what you are saying, then you would basically have some data like this:
parents
{
"_id": 1,
"xyz": ["abc", "abd", "abe", "abf"]
}
children
{ "_id": "abc", "fieldA": 123 },
{ "_id": "abd", "fieldA": 34 },
{ "_id": "abe", "fieldA": 123 },
{ "_id": "abf", "fieldA": 54 }
N.B. If you actually defined the parent reference within the child instead of an array of child references in the parent, then there is a listing example at the bottom. The same principles generally apply in either case however.
Where your current $lookup that produces a result like that in the question would be something like this:
{ "$lookup": {
"from": "children",
"localField": "xyz",
"foreignField": "_id"
"as": "xyz"
}}
Best Approach
Now you could do other operations on the array returned in order to actually return the total count and distinct counts, but there is a better way with any modern MongoDB release which you should be using. Namely there is a more expressive form of $lookup which allows a pipeline to be specified to act on the resulting children:
Parent.aggregate([
{ "$lookup": {
"from": "children",
"let": { "ids": "$xyz" },
"pipeline": [
{ "$match": {
"$expr": { "$in": [ "$_id", "$$ids" ] }
}},
{ "$group": {
"_id": "$fieldA",
"total": { "$sum": 1 }
}},
{ "$group": {
"_id": null,
"distinct": { "$sum": 1 },
"total": { "$sum": "$total" }
}}
],
"as": "xyz"
}},
{ "$addFields": {
"xyz": "$$REMOVE",
"distinctCount": { "$sum": "$xyz.distinct" },
"totalCount": { "$sum": "$xyz.total" }
}}
])
The whole point there being that you don't actually need all the array results to be returned from the $lookup, so instead of working with the returned array of all matching children you just reduce that content from within the pipeline expression of the $lookup.
In order to get a total count and a distinct count for the inner content, after the initial $match conditions which specify the "join" and what matches to return, you would then $group on the "distinct" value as the key and maintain a "count" of the elements found in total. The second $group uses a null value for the key since the only thing you want now is the count of the distinct keys already returned, and of course return the $sum of the existing total of counted elements.
The result being of course:
{
"_id": 1,
"distinctCount": 3,
"totalCount": 4
}
And since we are using $addFields this would be in addition to all other fields present in the parent document with the exception of xyz which we explicitly removed via the $$REMOVE operator.
You might also note the usage of $sum in that last stage. The actual result of our $lookup pipeline is of course a single document, but it is as always within an array, since that is what the output of $lookup always is. In this case it's just a very simple way ( being the shortest syntax ) to just extract those values from the array as individual fields in the parent document instead.
Alternate
The alternate approach is of course to just work with the returned array, and all this really needs is essentially any of the appropriate "set operators" and the $size operator:
Parent.aggregate([
{ "$lookup": {
"from": "children",
"localField": "xyz",
"foreignField": "_id",
"as": "xyz"
}},
{ "$addFields": {
"xyz": "$$REMOVE",
"distinctCount": { "$size": { "$setUnion": [ [], "$xyz.fieldA" ] }},
"totalCount": { "$size": "$xyz" }
}}
])
Here we use $setUnion basically providing arguments of an empty array [] and the array of fieldA values. Since this would return a "set" that is the combination of both arguments, the one thing that defines a "set" is that the values can appear only once and are thus *distinct. This is a quick way of obtaining only the distinct values and then of course each "array" ( or "set" ) is simply measured by $size for their respective counts.
So it "looks simple" but the problem is that it's not really efficient, and mostly because we spent operational time returning those array values from the $lookup and then we basically discarded the result. This is generally why the former approach is preferred since it will actually reduce the result before it is ever returned as an array. So "less work" overall.
If on the other hand you actually want to keep the array returned from the $lookup result then the latter case would be of course more desirable
Example listing
const { Schema } = mongoose = require('mongoose');
const uri = 'mongodb://localhost:27017/test';
const options = { useNewUrlParser: true, useUnifiedTopology: true };
mongoose.set('debug', true);
mongoose.set('useFindAndModify', false);
mongoose.set('useCreateIndex', true);
const parentSchema = new Schema({
_id: Number,
xyz: [{ type: String, ref: 'Child' }]
},{ _id: false });
parentSchema.index({ "xyz": 1 });
const childSchema = new Schema({
_id: String,
fieldA: Number
},{ _id: false });
const Parent = mongoose.model('Parent', parentSchema);
const Child = mongoose.model('Child', childSchema);
const log = data => console.log(JSON.stringify(data, undefined, 2));
(async function() {
try {
const conn = await mongoose.connect(uri, options);
// Clean data for demonstration
await Promise.all(
Object.values(conn.models).map(m => m.deleteMany())
);
// Insert some data
await Parent.create({ "_id": 1, "xyz": ["abc", "abd", "abe", "abf"] });
await Child.insertMany([
{ "_id": "abc", "fieldA": 123 },
{ "_id": "abd", "fieldA": 34 },
{ "_id": "abe", "fieldA": 123 },
{ "_id": "abf", "fieldA": 54 }
]);
let result1 = await Parent.aggregate([
{ "$lookup": {
"from": Child.collection.name,
"let": { "ids": "$xyz" },
"pipeline": [
{ "$match": {
"$expr": { "$in": [ "$_id", "$$ids" ] }
}},
{ "$group": {
"_id": "$fieldA",
"total": { "$sum": 1 }
}},
{ "$group": {
"_id": null,
"distinct": { "$sum": 1 },
"total": { "$sum": "$total" }
}}
],
"as": "xyz"
}},
{ "$addFields": {
"xyz": "$$REMOVE",
"distinctCount": { "$sum": "$xyz.distinct" },
"totalCount": { "$sum": "$xyz.total" }
}}
]);
log({ result1 });
let result2 = await Parent.aggregate([
{ "$lookup": {
"from": Child.collection.name,
"localField": "xyz",
"foreignField": "_id",
"as": "xyz"
}},
{ "$addFields": {
"xyz": "$$REMOVE",
"distinctCount": { "$size": { "$setUnion": [ [], "$xyz.fieldA" ] } },
"totalCount": { "$size": "$xyz" }
}}
]);
log({ result2 })
} catch(e) {
console.error(e);
} finally {
mongoose.disconnect();
}
})()
And the output:
Mongoose: parents.createIndex({ xyz: 1 }, { background: true })
Mongoose: parents.deleteMany({}, {})
Mongoose: children.deleteMany({}, {})
Mongoose: parents.insertOne({ xyz: [ 'abc', 'abd', 'abe', 'abf' ], _id: 1, __v: 0 }, { session: null })
Mongoose: children.insertMany([ { _id: 'abc', fieldA: 123, __v: 0 }, { _id: 'abd', fieldA: 34, __v: 0 }, { _id: 'abe', fieldA: 123, __v: 0 }, { _id: 'abf', fieldA: 54, __v: 0 }], {})
Mongoose: parents.aggregate([ { '$lookup': { from: 'children', let: { ids: '$xyz' }, pipeline: [ { '$match': { '$expr': { '$in': [ '$_id', '$$ids' ] } } }, { '$group': { _id: '$fieldA', total: { '$sum': 1 } } }, { '$group': { _id: null, distinct: { '$sum': 1 }, total: { '$sum': '$total' } } } ], as: 'xyz' } }, { '$addFields': { xyz: '$$REMOVE', distinctCount: { '$sum': '$xyz.distinct' }, totalCount: { '$sum': '$xyz.total' } } }], {})
{
"result1": [
{
"_id": 1,
"__v": 0,
"distinctCount": 3,
"totalCount": 4
}
]
}
Mongoose: parents.aggregate([ { '$lookup': { from: 'children', localField: 'xyz', foreignField: '_id', as: 'xyz' } }, { '$addFields': { xyz: '$$REMOVE', distinctCount: { '$size': { '$setUnion': [ [], '$xyz.fieldA' ] } }, totalCount: { '$size': '$xyz' } } }], {})
{
"result2": [
{
"_id": 1,
"__v": 0,
"distinctCount": 3,
"totalCount": 4
}
]
}
Example without child array in parent
Shows defining a schema without an array of values in the parent and instead defining the parent reference within all children:
const { Schema } = mongoose = require('mongoose');
const uri = 'mongodb://localhost:27017/test';
const options = { useNewUrlParser: true, useUnifiedTopology: true };
mongoose.set('debug', true);
mongoose.set('useFindAndModify', false);
mongoose.set('useCreateIndex', true);
const parentSchema = new Schema({
_id: Number,
},{ _id: false });
parentSchema.virtual("xyz", {
ref: 'Child',
localField: '_id',
foreignField: 'parent',
justOne: false
});
const childSchema = new Schema({
_id: String,
parent: Number,
fieldA: Number
},{ _id: false });
childSchema.index({ "parent": 1 });
const Parent = mongoose.model('Parent', parentSchema);
const Child = mongoose.model('Child', childSchema);
const log = data => console.log(JSON.stringify(data, undefined, 2));
(async function() {
try {
const conn = await mongoose.connect(uri, options);
// Clean data for demonstration
await Promise.all(
Object.values(conn.models).map(m => m.deleteMany())
);
// Insert some data
await Parent.create({ "_id": 1 });
await Child.insertMany([
{ "_id": "abc", "fieldA": 123 },
{ "_id": "abd", "fieldA": 34 },
{ "_id": "abe", "fieldA": 123 },
{ "_id": "abf", "fieldA": 54 }
].map(e => ({ ...e, "parent": 1 })));
let result1 = await Parent.aggregate([
{ "$lookup": {
"from": Child.collection.name,
"let": { "parent": "$_id" },
"pipeline": [
{ "$match": {
"$expr": { "$eq": [ "$parent", "$$parent" ] }
}},
{ "$group": {
"_id": "$fieldA",
"total": { "$sum": 1 }
}},
{ "$group": {
"_id": null,
"distinct": { "$sum": 1 },
"total": { "$sum": "$total" }
}}
],
"as": "xyz"
}},
{ "$addFields": {
"xyz": "$$REMOVE",
"distinctCount": { "$sum": "$xyz.distinct" },
"totalCount": { "$sum": "$xyz.total" }
}}
]);
log({ result1 });
let result2 = await Parent.aggregate([
{ "$lookup": {
"from": Child.collection.name,
"localField": "_id",
"foreignField": "parent",
"as": "xyz"
}},
{ "$addFields": {
"xyz": "$$REMOVE",
"distinctCount": { "$size": { "$setUnion": [ [], "$xyz.fieldA" ] } },
"totalCount": { "$size": "$xyz" }
}}
]);
log({ result2 })
} catch(e) {
console.error(e);
} finally {
mongoose.disconnect();
}
})()
And the output:
Mongoose: children.createIndex({ parent: 1 }, { background: true })
Mongoose: parents.deleteMany({}, {})
Mongoose: children.deleteMany({}, {})
Mongoose: parents.insertOne({ _id: 1, __v: 0 }, { session: null })
Mongoose: children.insertMany([ { _id: 'abc', fieldA: 123, parent: 1, __v: 0 }, { _id: 'abd', fieldA: 34, parent: 1, __v: 0 }, { _id: 'abe', fieldA: 123, parent: 1, __v: 0 }, { _id: 'abf', fieldA: 54, parent: 1, __v: 0 }], {})
Mongoose: parents.aggregate([ { '$lookup': { from: 'children', let: { parent: '$_id' }, pipeline: [ { '$match': { '$expr': { '$eq': [ '$parent', '$$parent' ] } } }, { '$group': { _id: '$fieldA', total: { '$sum': 1 } } }, { '$group': { _id: null, distinct: { '$sum': 1 }, total: { '$sum': '$total' } } } ], as: 'xyz' } }, { '$addFields': { xyz: '$$REMOVE', distinctCount: { '$sum': '$xyz.distinct' }, totalCount: { '$sum': '$xyz.total' } } }], {})
{
"result1": [
{
"_id": 1,
"__v": 0,
"distinctCount": 3,
"totalCount": 4
}
]
}
Mongoose: parents.aggregate([ { '$lookup': { from: 'children', localField: '_id', foreignField: 'parent', as: 'xyz' } }, { '$addFields': { xyz: '$$REMOVE', distinctCount: { '$size': { '$setUnion': [ [], '$xyz.fieldA' ] } }, totalCount: { '$size': '$xyz' } } }], {})
{
"result2": [
{
"_id": 1,
"__v": 0,
"distinctCount": 3,
"totalCount": 4
}
]
}
I ended up following first approach suggested by #Neil Lunn. Due to a fact that my schemas of parent and child are different from those assumed by #Neil Lunn I post my own answer whitch solves my particular problem:
Parent.aggregate([
{
$lookup: {
from: "children",
let: { id: "$_id" },
pipeline: [
{ $match: { $expr: { $eq: ["$x.id", "$$id"] } } },
{
$group: {
_id: "$fieldA",
count: { $sum: 1 }
}
},
{
$group: {
_id: null,
fieldA: { $sum: 1 },
count: { $sum: "$count" }
}
}
],
as: "children"
}
},
{
$project: {
total: { $sum: "$children.count" },
distinct: { $sum: "$children.fieldA" }
}
}
]);
I am trying to filter a document by a sub-documents referred property. Assume that I have already created models for each schema. The simplified schemas are the following:
const store = new Schema({
name: { type: String }
})
const price = new Schema({
price: { type: Number },
store: {
type: mongoose.Schema.Types.ObjectId,
ref: 'Store'
},
})
const product = new Schema({
name: {type: String},
prices: [{
type: mongoose.Schema.Types.ObjectId,
ref: 'Price'
}]
})
/*
Notation:
lowercase for schemas: product
uppercase for models: Product
*/
As a first approach I tried:
Product.find({'prices.store':storeId}).populate('prices')
but this does not work as filtering by a sub-document property is not supported on mongoose.
My current approach is using the aggregation framework. This is how the aggregation looks:
{
$unwind: '$prices'
},
{
$lookup: {
from: 'prices',
localField: 'prices',
foreignField: '_id',
as: 'prices'
}
},
{
$unwind: '$prices'
},
{
$lookup: {
from: 'stores',
localField: 'prices.store',
foreignField: '_id',
as: 'prices.store'
}
}, // populate
{
$match: {
'prices.store._id': new mongoose.Types.ObjectId(storeId)
}
}, // filter by store id
{ $group: { _id: '$id', doc: { $first: '$$ROOT' } } },
{ $replaceRoot: { newRoot: '$doc' } }
// Error occurs in $group & $replaceRoot
For example, before the last two stages if the record being saved is:
{
name: 'Milk',
prices: [
{store: 1, price: 3.2},
{store: 2, price: 4.0}
]
}
then the aggregation returned: (notice the product is the same but displaying each price in different results)
[
{
id: 4,
name: 'Milk',
prices: {
id: 10,
store: { _id: 1, name : 'Walmart' },
price: 3.2
}
},
{
id: 4,
name: 'Milk',
prices: {
id: 11,
store: { _id: 2, name : 'CVS' },
price: 4.0
},
}
]
To solve this issue I added the last part:
{ $group: { _id: '$id', doc: { $first: '$$ROOT' } } },
{ $replaceRoot: { newRoot: '$doc' } }
But this last part only returns the following:
{
id: 4,
name: 'Milk',
prices: {
id: 10,
store: { _id: 1, name : 'Walmart' },
price: 3.2
}
}
Now prices is an object, it should be an array and it should contain all prices (2 in this case).
Question
How to return all prices (as an array) with the store field populated and filtered by storeId?
Expected result:
{
id: 4,
name: 'Milk',
prices: [
{
id: 10,
store: { _id: 1, name : 'Walmart' },
price: 3.2
},
{
id: 11,
store: { _id: 2, name : 'CVS' },
price: 4.0
}]
}
EDIT
I want to filter products that contain prices in a given store. It should return the product with its prices, all of them.
I'm not totally convinced your existing pipeline is the most optimal, but without sample data to work from it's hard to really tell otherwise. So just working onward from what you have:
Using $unwind
var pipeline = [
// { $unwind: '$prices' }, // note: should not need this past MongoDB 3.0
{ $lookup: {
from: 'prices',
localField: 'prices',
foreignField: '_id',
as: 'prices'
}},
{ $unwind: '$prices' },
{ $lookup: {
from: 'stores',
localField: 'prices.store',
foreignField: '_id',
as: 'prices.store'
}},
// Changes from here
{ $unwind: '$prices.store' },
{ $match: {'prices.store._id': mongoose.Types.ObjectId(storeId) } },
{ $group: {
_id: '$_id',
name: { $first: '$name' },
prices: { $push: '$prices' }
}}
];
The points there start with:
Initial $unwind - Should not be required. Only in very early MongoDB 3.0 releases was this ever a requirement to $unwind an array of values before using $lookup on those values.
$unwind after $lookup - Is always required if you expect a "singular" object as matching, since $lookup always returns an array.
$match after $unwind - Is actually an "optimization" for pipeline processing and in fact a requirement in order to "filter". Without $unwind it's just a verification that "something is there" but items that did not match would not be removed.
$push in $group - This is the actual part the re-builds the "prices"array.
The key point you were basically missing was using $first for the "whole document" content. You really don't ever want that, and even if you want more than just "name" you always want to $push the "prices".
In fact you probably do want more fields than just name from the original document, but really you should therefore be using the following form instead.
Expressive $lookup
An alternate is available with most modern MongoDB releases since MongoDB 3.6, which frankly you should be using at minimum:
var pipeline = [
{ $lookup: {
from: 'prices',
let: { prices: '$prices' },
pipeline: [
{ $match: {
store: mongoose.Types.ObjectId(storeId),
$expr: { $in: [ '$_id', '$$prices' ] }
}},
{ $lookup: {
from: 'stores',
let: { store: '$store' },
pipeline: [
{ $match: { $expr: { $eq: [ '$_id', '$$store' ] } }
],
as: 'store'
}},
{ $unwind: '$store' }
],
as: 'prices'
}},
// remove results with no matching prices
{ $match: { 'prices.0': { $exists: true } } }
];
So the first thing to notice there is the "outer" pipeline is actually just a single $lookup stage, since all it really needs to do is "join" to the prices collection. From the perspective of joining to your original collection this is also true since the additional $lookup in the above example is actually related from prices to another collection.
This is then exactly what this new form does, so instead of using $unwind on the resulting array and then following on the join, only the matching items for "prices" are then "joined" to the "stores" collection, and before those are returned into the array. Of course since there is a "one to one" relationship with the "store", this will actually $unwind.
In short, the output of this simply has the original document with a "prices" array inside it. So there is no need to re-construct via $group and no confusion of what you use $first on and what you $push.
NOTE: I'm more than a little suspect of your "filter stores" statement and attempting to match the store field as presented in the "prices" collection. The question shows expected output from two different stores even though you specify an equality match.
If anything I suspect you might mean a "list of stores", which would instead be more like:
store: { $in: storeList.map(store => mongoose.Types.ObjectId(store)) }
Which is how you would work with a "list of strings" in both cases, using $in for matching against a "list" and the Array.map() to work with a supplied list and return each as ObjectId() values.
TIP: With mongoose you use a "model" rather than working with collection names, and the actual MongoDB collection names is typically the plural of the model name you registered.
So you don't have to "hardcode" the actual collection names for $lookup, simply use:
Model.collection.name
The .collection.name is an accessible property on all models, and can save you the trouble of remembering to actually name the collection for $lookup. It also protects you should you ever change your mongoose.model() instance registration in a way which alters the stored collection name with MongoDB.
Full Demonstration
The following is a self contained listing demonstrating both approaches as work and how they produce the same results:
const { Schema, Types: { ObjectId } } = mongoose = require('mongoose');
const uri = 'mongodb://localhost:27017/shopping';
const opts = { useNewUrlParser: true };
mongoose.set('useFindAndModify', false);
mongoose.set('useCreateIndex', true);
mongoose.set('debug', true);
const storeSchema = new Schema({
name: { type: String }
});
const priceSchema = new Schema({
price: { type: Number },
store: { type: Schema.Types.ObjectId, ref: 'Store' }
});
const productSchema = new Schema({
name: { type: String },
prices: [{ type: Schema.Types.ObjectId, ref: 'Price' }]
});
const Store = mongoose.model('Store', storeSchema);
const Price = mongoose.model('Price', priceSchema);
const Product = mongoose.model('Product', productSchema);
const log = data => console.log(JSON.stringify(data, undefined, 2));
(async function() {
try {
const conn = await mongoose.connect(uri, opts);
// Clean data
await Promise.all(
Object.entries(conn.models).map(([k, m]) => m.deleteMany())
);
// Insert working data
let [StoreA, StoreB, StoreC] = await Store.insertMany(
["StoreA", "StoreB", "StoreC"].map(name => ({ name }))
);
let [PriceA, PriceB, PriceC, PriceD, PriceE, PriceF]
= await Price.insertMany(
[[StoreA,1],[StoreB,2],[StoreA,3],[StoreC,4],[StoreB,5],[StoreC,6]]
.map(([store, price]) => ({ price, store }))
);
let [Milk, Cheese, Bread] = await Product.insertMany(
[
{ name: 'Milk', prices: [PriceA, PriceB] },
{ name: 'Cheese', prices: [PriceC, PriceD] },
{ name: 'Bread', prices: [PriceE, PriceF] }
]
);
// Test 1
{
log("Single Store - expressive")
const pipeline = [
{ '$lookup': {
'from': Price.collection.name,
'let': { prices: '$prices' },
'pipeline': [
{ '$match': {
'store': ObjectId(StoreA._id), // demo - it's already an ObjectId
'$expr': { '$in': [ '$_id', '$$prices' ] }
}},
{ '$lookup': {
'from': Store.collection.name,
'let': { store: '$store' },
'pipeline': [
{ '$match': { '$expr': { '$eq': [ '$_id', '$$store' ] } } }
],
'as': 'store'
}},
{ '$unwind': '$store' }
],
as: 'prices'
}},
{ '$match': { 'prices.0': { '$exists': true } } }
];
let result = await Product.aggregate(pipeline);
log(result);
}
// Test 2
{
log("Dual Store - expressive");
const pipeline = [
{ '$lookup': {
'from': Price.collection.name,
'let': { prices: '$prices' },
'pipeline': [
{ '$match': {
'store': { '$in': [StoreA._id, StoreB._id] },
'$expr': { '$in': [ '$_id', '$$prices' ] }
}},
{ '$lookup': {
'from': Store.collection.name,
'let': { store: '$store' },
'pipeline': [
{ '$match': { '$expr': { '$eq': [ '$_id', '$$store' ] } } }
],
'as': 'store'
}},
{ '$unwind': '$store' }
],
as: 'prices'
}},
{ '$match': { 'prices.0': { '$exists': true } } }
];
let result = await Product.aggregate(pipeline);
log(result);
}
// Test 3
{
log("Single Store - legacy");
const pipeline = [
{ '$lookup': {
'from': Price.collection.name,
'localField': 'prices',
'foreignField': '_id',
'as': 'prices'
}},
{ '$unwind': '$prices' },
// Alternately $match can be done here
// { '$match': { 'prices.store': StoreA._id } },
{ '$lookup': {
'from': Store.collection.name,
'localField': 'prices.store',
'foreignField': '_id',
'as': 'prices.store'
}},
{ '$unwind': '$prices.store' },
{ '$match': { 'prices.store._id': StoreA._id } },
{ '$group': {
'_id': '$_id',
'name': { '$first': '$name' },
'prices': { '$push': '$prices' }
}}
];
let result = await Product.aggregate(pipeline);
log(result);
}
// Test 4
{
log("Dual Store - legacy");
const pipeline = [
{ '$lookup': {
'from': Price.collection.name,
'localField': 'prices',
'foreignField': '_id',
'as': 'prices'
}},
{ '$unwind': '$prices' },
// Alternately $match can be done here
{ '$match': { 'prices.store': { '$in': [StoreA._id, StoreB._id] } } },
{ '$lookup': {
'from': Store.collection.name,
'localField': 'prices.store',
'foreignField': '_id',
'as': 'prices.store'
}},
{ '$unwind': '$prices.store' },
//{ '$match': { 'prices.store._id': { '$in': [StoreA._id, StoreB._id] } } },
{ '$group': {
'_id': '$_id',
'name': { '$first': '$name' },
'prices': { '$push': '$prices' }
}}
];
let result = await Product.aggregate(pipeline);
log(result);
}
} catch(e) {
console.error(e);
} finally {
mongoose.disconnect();
}
})()
Which produces the output:
Mongoose: stores.deleteMany({}, {})
Mongoose: prices.deleteMany({}, {})
Mongoose: products.deleteMany({}, {})
Mongoose: stores.insertMany([ { _id: 5c7c79bcc78675135c09f54b, name: 'StoreA', __v: 0 }, { _id: 5c7c79bcc78675135c09f54c, name: 'StoreB', __v: 0 }, { _id: 5c7c79bcc78675135c09f54d, name: 'StoreC', __v: 0 } ], {})
Mongoose: prices.insertMany([ { _id: 5c7c79bcc78675135c09f54e, price: 1, store: 5c7c79bcc78675135c09f54b, __v: 0 }, { _id: 5c7c79bcc78675135c09f54f, price: 2, store: 5c7c79bcc78675135c09f54c, __v: 0 }, { _id: 5c7c79bcc78675135c09f550, price: 3, store: 5c7c79bcc78675135c09f54b, __v: 0 }, { _id: 5c7c79bcc78675135c09f551, price: 4, store: 5c7c79bcc78675135c09f54d, __v: 0 }, { _id: 5c7c79bcc78675135c09f552, price: 5, store: 5c7c79bcc78675135c09f54c, __v: 0 }, { _id: 5c7c79bcc78675135c09f553, price: 6, store: 5c7c79bcc78675135c09f54d, __v: 0 } ], {})
Mongoose: products.insertMany([ { prices: [ 5c7c79bcc78675135c09f54e, 5c7c79bcc78675135c09f54f ], _id: 5c7c79bcc78675135c09f554, name: 'Milk', __v: 0 }, { prices: [ 5c7c79bcc78675135c09f550, 5c7c79bcc78675135c09f551 ], _id: 5c7c79bcc78675135c09f555, name: 'Cheese', __v: 0 }, { prices: [ 5c7c79bcc78675135c09f552, 5c7c79bcc78675135c09f553 ], _id: 5c7c79bcc78675135c09f556, name: 'Bread', __v: 0 } ], {})
"Single Store - expressive"
Mongoose: products.aggregate([ { '$lookup': { from: 'prices', let: { prices: '$prices' }, pipeline: [ { '$match': { store: 5c7c79bcc78675135c09f54b, '$expr': { '$in': [ '$_id', '$$prices' ] } } }, { '$lookup': { from: 'stores', let: { store: '$store' }, pipeline: [ { '$match': { '$expr': { '$eq': [ '$_id', '$$store' ] } } } ], as: 'store' } }, { '$unwind': '$store' } ], as: 'prices' } }, { '$match': { 'prices.0': { '$exists': true } } } ], {})
[
{
"_id": "5c7c79bcc78675135c09f554",
"prices": [
{
"_id": "5c7c79bcc78675135c09f54e",
"price": 1,
"store": {
"_id": "5c7c79bcc78675135c09f54b",
"name": "StoreA",
"__v": 0
},
"__v": 0
}
],
"name": "Milk",
"__v": 0
},
{
"_id": "5c7c79bcc78675135c09f555",
"prices": [
{
"_id": "5c7c79bcc78675135c09f550",
"price": 3,
"store": {
"_id": "5c7c79bcc78675135c09f54b",
"name": "StoreA",
"__v": 0
},
"__v": 0
}
],
"name": "Cheese",
"__v": 0
}
]
"Dual Store - expressive"
Mongoose: products.aggregate([ { '$lookup': { from: 'prices', let: { prices: '$prices' }, pipeline: [ { '$match': { store: { '$in': [ 5c7c79bcc78675135c09f54b, 5c7c79bcc78675135c09f54c ] }, '$expr': { '$in': [ '$_id', '$$prices' ] } } }, { '$lookup': { from: 'stores', let: { store: '$store' }, pipeline: [ { '$match': { '$expr': { '$eq': [ '$_id', '$$store' ] } } } ], as: 'store' } }, { '$unwind': '$store' } ], as: 'prices' } }, { '$match': { 'prices.0': { '$exists': true } } } ], {})
[
{
"_id": "5c7c79bcc78675135c09f554",
"prices": [
{
"_id": "5c7c79bcc78675135c09f54e",
"price": 1,
"store": {
"_id": "5c7c79bcc78675135c09f54b",
"name": "StoreA",
"__v": 0
},
"__v": 0
},
{
"_id": "5c7c79bcc78675135c09f54f",
"price": 2,
"store": {
"_id": "5c7c79bcc78675135c09f54c",
"name": "StoreB",
"__v": 0
},
"__v": 0
}
],
"name": "Milk",
"__v": 0
},
{
"_id": "5c7c79bcc78675135c09f555",
"prices": [
{
"_id": "5c7c79bcc78675135c09f550",
"price": 3,
"store": {
"_id": "5c7c79bcc78675135c09f54b",
"name": "StoreA",
"__v": 0
},
"__v": 0
}
],
"name": "Cheese",
"__v": 0
},
{
"_id": "5c7c79bcc78675135c09f556",
"prices": [
{
"_id": "5c7c79bcc78675135c09f552",
"price": 5,
"store": {
"_id": "5c7c79bcc78675135c09f54c",
"name": "StoreB",
"__v": 0
},
"__v": 0
}
],
"name": "Bread",
"__v": 0
}
]
"Single Store - legacy"
Mongoose: products.aggregate([ { '$lookup': { from: 'prices', localField: 'prices', foreignField: '_id', as: 'prices' } }, { '$unwind': '$prices' }, { '$lookup': { from: 'stores', localField: 'prices.store', foreignField: '_id', as: 'prices.store' } }, { '$unwind': '$prices.store' }, { '$match': { 'prices.store._id': 5c7c79bcc78675135c09f54b } }, { '$group': { _id: '$_id', name: { '$first': '$name' }, prices: { '$push': '$prices' } } } ], {})
[
{
"_id": "5c7c79bcc78675135c09f555",
"name": "Cheese",
"prices": [
{
"_id": "5c7c79bcc78675135c09f550",
"price": 3,
"store": {
"_id": "5c7c79bcc78675135c09f54b",
"name": "StoreA",
"__v": 0
},
"__v": 0
}
]
},
{
"_id": "5c7c79bcc78675135c09f554",
"name": "Milk",
"prices": [
{
"_id": "5c7c79bcc78675135c09f54e",
"price": 1,
"store": {
"_id": "5c7c79bcc78675135c09f54b",
"name": "StoreA",
"__v": 0
},
"__v": 0
}
]
}
]
"Dual Store - legacy"
Mongoose: products.aggregate([ { '$lookup': { from: 'prices', localField: 'prices', foreignField: '_id', as: 'prices' } }, { '$unwind': '$prices' }, { '$match': { 'prices.store': { '$in': [ 5c7c79bcc78675135c09f54b, 5c7c79bcc78675135c09f54c ] } } }, { '$lookup': { from: 'stores', localField: 'prices.store', foreignField: '_id', as: 'prices.store' } }, { '$unwind': '$prices.store' }, { '$group': { _id: '$_id', name: { '$first': '$name' }, prices: { '$push': '$prices' } } } ], {})
[
{
"_id": "5c7c79bcc78675135c09f555",
"name": "Cheese",
"prices": [
{
"_id": "5c7c79bcc78675135c09f550",
"price": 3,
"store": {
"_id": "5c7c79bcc78675135c09f54b",
"name": "StoreA",
"__v": 0
},
"__v": 0
}
]
},
{
"_id": "5c7c79bcc78675135c09f556",
"name": "Bread",
"prices": [
{
"_id": "5c7c79bcc78675135c09f552",
"price": 5,
"store": {
"_id": "5c7c79bcc78675135c09f54c",
"name": "StoreB",
"__v": 0
},
"__v": 0
}
]
},
{
"_id": "5c7c79bcc78675135c09f554",
"name": "Milk",
"prices": [
{
"_id": "5c7c79bcc78675135c09f54e",
"price": 1,
"store": {
"_id": "5c7c79bcc78675135c09f54b",
"name": "StoreA",
"__v": 0
},
"__v": 0
},
{
"_id": "5c7c79bcc78675135c09f54f",
"price": 2,
"store": {
"_id": "5c7c79bcc78675135c09f54c",
"name": "StoreB",
"__v": 0
},
"__v": 0
}
]
}
]