Related
I am trying to do a lookup from multiple references
Here is a Mongo Playground
Here is my data
Insp
The Insp document contains an array of references to Users (by user ID)
[
{
"_id": {
"$oid": "6359a12fb9450da3d8d8cdd2"
},
"REF_Users": [
{
"$oid": "6359a0f1b9450da3d8d8cdc7"
},
{
"$oid": "6359a070f1e84209e0c78fc2"
}
],
"name": "Once"
}
]
Users
The Users document contains information about a user and it has a reference to the UserType (by userType ID)
[
{
"_id": {
"$oid": "6359a070f1e84209e0c78fc2"
},
"REF_UserType": {
"$oid": "63596323b679475de490500a"
},
"fName": "Billy"
},
{
"_id": {
"$oid": "6359a0f1b9450da3d8d8cdc7"
},
"REF_UserType": {
"$oid": "63596323b679475de4905007"
},
"fName": "Mike"
}
]
UserType
The UserType document holds type information
[
{
"_id": {
"$oid": "63596323b679475de4905007"
},
"value": 100,
"name": "INS"
},
{
"_id": {
"$oid": "63596323b679475de490500a"
},
"value": 200,
"name": "CLS"
}
]
Expected output
I want the userType for each user to be with the respective user
{
"_id": "6359a12fb9450da3d8d8cdd2",
"people": [
{
"_id": "6359a070f1e84209e0c78fc2",
"userType": {
"_id": "63596323b679475de490500a",
"value": 200,
"name": "CLS"
},
"fName": "Billy"
},
{
"_id": "6359a0f1b9450da3d8d8cdc7",
"userType": {
"_id": "63596323b679475de4905007",
"value": 100,
"name": "INS"
},
"fName": "Mike"
}
]
}
TRY 1
This is my pipeline so far
[
{
"$match": {}
},
{
"$lookup": {
"from": "users",
"localField": "REF_Users",
"foreignField": "_id",
"as": "people"
}
},
{
"$lookup": {
"from": "usertypes",
"localField": "people.REF_UserType",
"foreignField": "_id",
"as": "userType"
}
},
{
"$project": {
"REF_Users": 0,
"people.REF_UserType": 0
}
}
]
Result of TRY 1
{
"_id": "6359a12fb9450da3d8d8cdd2",
"people": [
{
"_id": "6359a070f1e84209e0c78fc2",
"fName": "Billy"
},
{
"_id": "6359a0f1b9450da3d8d8cdc7",
"fName": "Mike"
}
],
"userType": [
{
"_id": "63596323b679475de4905007",
"value": 100,
"name": "INS"
},
{
"_id": "63596323b679475de490500a",
"value": 200,
"name": "CLS"
}
]
}
It works in Compass...
It works in the playground
When I put the code into NodeJS and run it from my server:
TRY 2
const agg_project_try = {
people: {
$map: {
input: '$people',
as: 'people',
in: {
$mergeObjects: [
'$$people',
{
userType: {
$first: {
$filter: {
input: '$userType',
cond: {
$eq: ['$$people.REF_UserType', '$$this._id'],
},
},
},
},
},
],
},
},
},
};
I get this error
Arguments must be aggregate pipeline operators
TRY 3
I exported from Compass as NODE
[
{
'$lookup': {
'from': 'users',
'localField': 'REF_Users',
'foreignField': '_id',
'as': 'people'
}
}, {
'$lookup': {
'from': 'usertypes',
'localField': 'people.REF_UserType',
'foreignField': '_id',
'as': 'userType'
}
}, {
'$project': {
'people': {
'$map': {
'input': '$people',
'as': 'people',
'in': {
'$mergeObjects': [
'$$people', {
'userType': {
'$first': {
'$filter': {
'input': '$userType',
'cond': {
'$eq': [
'$$people.REF_UserType', '$$this._id'
]
}
}
}
}
}
]
}
}
}
}
}, {
'$unset': 'people.REF_UserType'
}
]
Then tried the 'project' portion in my server
const agg_project_try = {
'people': {
'$map': {
'input': '$people',
'as': 'people',
'in': {
'$mergeObjects': [
'$$people', {
'userType': {
'$first': {
'$filter': {
'input': '$userType',
'cond': {
'$eq': [
'$$people.REF_UserType', '$$this._id'
]
}
}
}
}
}
]
}
}
}
};
I get this error
Arguments must be aggregate pipeline operators
Here is my node JS pipeline ( that causes the error )
[
{ "$match": {} },
{ "$lookup": { "from": "users", "localField": "REF_Users", "foreignField": "_id", "as": "people" } },
{ "$lookup": { "from": "usertypes", "localField": "people.REF_UserType", "foreignField": "_id", "as": "userType" } },
{
"people": {
"$map": {
"input": "$people",
"as": "people",
"in": {
"$mergeObjects": [
"$$people",
{
"userType": {
"$first": {
"$filter": { "input": "$userType", "cond": { "$eq": ["$$people.REF_UserType", "$$this._id"] } }
}
}
}
]
}
}
}
},
{ "$project": { "REF_Users": 0 } }
]
ANSWER
Up too late last night working on this stuff, actually need the "project" statement to do a projection - doh !
$project:{
'people': {
$map: {
input: '$peopleLookup',
as: 'tempPeople',
in: {
$mergeObjects: [
'$$tempPeople',
{
'userType': {
$first: {
$filter: {
input: '$userTypeLookup',
cond: {
$eq: ['$$tempPeople.REF_UserType', '$$this._id'],
},
},
},
},
},
],
},
},
},
}
Thank you!
In $project stage, you need to iterate each document from the people array`.
Merge ($merge) the current iterated document with the first ($first) filtered ($filter) result from the userType array.
db.workoutDetailSchema.aggregate([
{
"$match": {}
},
{
"$lookup": {
"from": "users",
"localField": "REF_Users",
"foreignField": "_id",
"as": "people"
}
},
{
"$lookup": {
"from": "usertypes",
"localField": "people.REF_UserType",
"foreignField": "_id",
"as": "userType"
}
},
{
"$project": {
"people": {
$map: {
input: "$people",
as: "people",
in: {
$mergeObjects: [
"$$people",
{
userType: {
$first: {
$filter: {
input: "$userType",
cond: {
$eq: [
"$$people.REF_UserType",
"$$this._id"
]
}
}
}
}
}
]
}
}
}
}
},
{
$unset: "people.REF_UserType"
}
])
Demo # Mongo Playground
I just merged the documents using javascript.
Demo#mongoplayground
db.workoutDetailSchema.aggregate([
{
"$match": {}
},
{
"$lookup": {
"from": "users",
"localField": "REF_Users",
"foreignField": "_id",
"as": "peoples"
}
},
{
"$lookup": {
"from": "usertypes",
"localField": "peoples.REF_UserType",
"foreignField": "_id",
"as": "userType"
}
},
{
$addFields: {
people: {
$function: {
body: "function (people, userType) {people.forEach(function (item, index) {if(JSON.stringify(userType[index]._id) === JSON.stringify(item.REF_UserType)){people[index].userType=userType[index];}});return people;}",
args: [
"$peoples",
"$userType"
],
lang: "js"
},
}
}
},
{
"$project": {
"REF_Users": 0,
"peoples": 0,
"userType": 0,
"people.REF_UserType": 0,
}
}
])
Output
[
{
"_id": ObjectId("6359a12fb9450da3d8d8cdd2"),
"name": "Once",
"people": [
{
"_id": ObjectId("6359a0f1b9450da3d8d8cdc7"),
"fName": "Mike",
"userType": {
"_id": ObjectId("63596323b679475de4905007"),
"name": "INS",
"value": 100
}
},
{
"_id": ObjectId("6359a070f1e84209e0c78fc2"),
"fName": "Billy",
"userType": {
"_id": ObjectId("63596323b679475de490500a"),
"name": "CLS",
"value": 200
}
}
]
}
]
I have a Ranks collection with documents which looks like this:
[
{
"_id": "1",
"url": "ex1.com",
"keyword": "k1",
"rank": 19,
"createdAt": "2021-06-02",
"user": "616c542660d23fc17469b47e"
},
{
"_id": "2",
"url": "ex1.com",
"keyword": "k1",
"rank": 14,
"createdAt": "2021-06-01",
"user": "616c542660d23fc17469b47e"
},
{
"_id": "3",
"url": "ex1.com",
"keyword": "k2",
"rank": 8,
"createdAt": "2021-05-01",
"user": "616c542660d23fc17469b47e"
},
{
"_id": "4",
"url": "ex2.com",
"keyword": "k3",
"rank": 4,
"createdAt": "2021-05-01",
"user": "616c542660d23fc17469b47e"
}
]
users collection with documents which looks like this:
[
{
_id: "616c542660d23fc17469b47e",
email: "some#email.com"
}
]
I'm trying to run an aggregation which will return each user object + user's data array that grouped by url, each url object has keywords array that includes unique and last (by date) rank keyword
This is what I tried but the query returns all url's keywords, how can i make it return unique and last (by createdAt date) keywords
Rank.aggregate([
{
$match: {}
},
{
$lookup: {
from: 'users',
localField: 'user',
foreignField: '_id',
as: 'user'
}
},
{
$project: {
user: {
$arrayElemAt: ['$user', 0]
},
url: '$url',
keyword: '$keyword',
rank: '$rank',
createdAt: '$createdAt',
}
},
{
$sort: {
createdAt: -1
}
},
{
$group: {
_id: '$user._id',
user: {
$first: '$user'
},
data: {
$push: {
id: '$_id',
url: '$url',
keyword: '$keyword',
rank: '$rank',
createdAt: '$createdAt',
}
}
}
}
])
Expected output:
[{
user: {
_id: "616c542660d23fc17469b47e",
email: "some#email.com"
},
data: [
{
url: "ex1.com",
keywords: [
{
keyword: "k1",
rank: 19,
createdAt: "2021-06-02",
},
{
keyword: "k2",
rank: 8,
createdAt: "2021-05-01"
},
]
},
{
url: "ex2.com",
keywords: [
{
keyword: "k3",
rank: 4,
createdAt: "2021-05-01"
},
]
}
]
}]
Here it is the solution that I came out with. Playground
Full explanation:
We group by "$url","$user" and "$keyword" to get the unique combinations of this fields. AT this point waht we want is only the unique keywords, but we have to use the user and url fields, becouse we would groupBy those later too.Because we order them by createdAt, if we get the first document it will be the last one created.
{
"$sort": {
"createdAt": 1
}
},
{
"$group": {
"_id": [
"$url",
"$user",
"$keyword"
],
"keywords": {
$first: "$$ROOT"
}
}
},
Then we will format this keyword information a bit to group it by url. This step will give us the keywords per URL.
{
"$project": {
"url": "$keywords.url",
"user": "$keywords.user",
"keywords": "$keywords",
"_id": 0
}
},
{
"$group": {
"_id": [
"$user",
"$url"
],
"data": {
$push: "$$ROOT"
}
}
},
Finally we will group the URLs by user. Notice that we have grouped by URL and by user in each groupBy in order to not lose those fields.
{
"$project": {
"url": {
$first: "$data.keywords.url"
},
"user": {
$first: "$data.keywords.user"
},
"keywords": "$data.keywords",
"_id": 0
}
},
{
"$group": {
"_id": "$user",
"data": {
$push: "$$ROOT"
}
}
},
At this step we have almost all the information we needed grouped together. We would perform a lookUp to get the email from the Users collection and do the final mapping to remove some redundant data.
{
$lookup: {
from: "users",
localField: "_id",
foreignField: "_id",
as: "user"
}
},
{
"$unwind": "$user"
},
{
"$project": {
"_id": 0,
"data.user": 0,
"data.keywords._id": 0,
"data.keywords.url": 0,
"data.keywords.user": 0
}
},
I currently have 3 different collections.
ColA
{
id: 1,
some_other_data: "fff"
}
ColB
{
id: 1,
colA_id: 1,
spec_id: 5,
data: "test"
}
and
ColC
{
id: 5,
colA_id: 1,
name: "xxx"
}
My current code:
const list = await ColA.aggregate([
{
$lookup: {
from: "ColB",
localField: "_id",
foreignField: "ColA_id",
as: "col_b_list",
},
},
{
$lookup: {
from: "ColC",
localField: "col_b_list.spec_id",
foreignField: "_id",
as: "col_c_list",
},
},
{
$project: {
_id: 1,
list: "$col_b_list",
},
},
]);
return list;
I have following output.
[
{
"_id": "6123858b5b8dcc0e749c9e39",
"list": [
{
"_id": "6123858b5b8dcc0e749c9e43",
"spec_id": "60d33125f81840c010052e03",
"createdAt": "2021-08-23T11:24:59.292Z",
"updatedAt": "2021-08-23T11:24:59.292Z",
"__v": 0
},
{
"_id": "612386317dd1cb0ebcef1862",
"spec_id": "60d33125f81840c010052e03",
"createdAt": "2021-08-23T11:27:45.515Z",
"updatedAt": "2021-08-23T11:27:45.515Z",
"__v": 0
}
]
}
]
What I'm trying to achieve is to get spec_id in ColB, and the id from ColC together in one object.
But what I want is, that in the list object the looked up Collection ColC is included, like:
[
{
"_id": "6123858b5b8dcc0e749c9e39",
"some_other_data": "fff",
"list": [
{
"_id": "6123858b5b8dcc0e749c9e43",
"spec_id": "60d33125f81840c010052e03",
"name": "xxx",
"createdAt": "2021-08-23T11:24:59.292Z",
"updatedAt": "2021-08-23T11:24:59.292Z",
"__v": 0
},
{
"_id": "612386317dd1cb0ebcef1862",
"spec_id": "60d33125f81840c010052e03",
"name": "yyy",
"createdAt": "2021-08-23T11:27:45.515Z",
"updatedAt": "2021-08-23T11:27:45.515Z",
"__v": 0
}
]
}
]
I tried to map it in the project stage, but somehow there where multiple same objects, so I could not get it working properly.
Thanks for the help in advance!
You can use nested $lookup, using lookup with aggregation pipeline,
const list = await ColA.aggregate([
{
$lookup: {
from: "ColB",
let: { id: "$_id" },
pipeline: [
{
$match: {
$expr: { $eq: ["$$id", "$ColA_id"] }
}
},
{
$lookup: {
from: "ColC",
localField: "spec_id",
foreignField: "id",
as: "ColC"
}
},
{
$addFields: {
ColC: { $arrayElemAt: ["$ColC", 0] }
}
},
// { $project: {} }
],
as: "list"
}
}
])
I am trying to sort by the task._id & date in desc order. I am able to sort by task._id but sortibg bydate doesnt work, I tried changing the order in aggregate still no luck. I get the response but just the order by usertasks were added in collection and not by the usertask.date
User(name, address, etc)
Task(name, icon, assignee)
UserTask(User.ObjectId, Task.ObjectId, date)
User Collection:
{
"users": [
{
"name": "Bill",
"phone": "345"
},
{
"name": "Steve",
"phone": "123"
},
{
"name": "Elon",
"phone": "567"
}
]
}
Task collection:
{
"tasks": [
{
"name": "Run 100m",
"icon": "run",
"assignee": "Elon"
},
{
"name": "Walk 1 hour",
"icon": "walk",
"assignee": "Bill"
},
{
"name": "Jog 30 minutes",
"icon": "jog",
"assignee": "Steve"
}
]
}
UserTasks:
{
"_id": "5e72fec..",
"user": "5e72fa4..",
"task": "5e72fbac..",
"date": "2020-03-03T05:10:10.000Z",
"createdAt": "2020-03-19T05:10:37.027Z",
"updatedAt": "2020-03-19T05:10:37.027Z",
"__v": 0
},
{
"_id": "5e72fed3..",
"user": "5e72fa4e..",
"task": "5e72fbac..",
"date": "2020-03-12T05:10:10.000Z",
"createdAt": "2020-03-19T05:10:43.296Z",
"updatedAt": "2020-03-19T05:10:43.296Z",
"__v": 0
},
{
"_id": "5e72fed6..",
"user": "5e72fa..",
"task": "5e72fb..",
"date": "2020-03-15T05:10:10.000Z",
"createdAt": "2020-03-19T05:10:46.057Z",
"updatedAt": "2020-03-19T05:10:46.057Z",
"__v": 0
},
{
"_id": "5e72feda...",
"user": "5e72fa4..",
"task": "5e72fb..",
"date": "2020-03-07T05:10:10.000Z",
"createdAt": "2020-03-19T05:10:50.785Z",
"updatedAt": "2020-03-19T05:10:50.785Z",
"__v": 0
}
This is the Aggregate that needs changing
UserTask.aggregate([
{
$lookup: {
from: "tasks",
localField: "task",
foreignField: "_id",
as: "matchedTask"
}
},
{
$unwind: "$matchedTask"
},
{
$lookup: {
from: "users",
localField: "user",
foreignField: "_id",
as: "matchedUser"
}
},
{
$unwind: "$matchedUser"
},
{
$group: {
_id: "$matchedTask._id",
name: {$first: "$matchedTask.name"},
icon: {$first: "$matchedTask.icon"},
assignee: { $first: "$matchedTask.assignee" },
userdata: {
$push: {
name: "$matchedUser.name",
date: "$date"
}
}
}
},
{
$sort: { _id: 1, "userdata.date": -1 }
}
])
.exec()
.then(doc => res.status(200).json(doc))
.catch(err => res.status(400).json("Error: " + err));
The response is shown below, please note the usertask.date. it is NOT sorted
{
"_id": "5e...",
"name": "Run 100m",
"icon": "run",
"assignee": "Elon",
"userdata": [
{
"name": "Elon",
"date": "2020-03-21T20:02:38.143Z"
},
{
"name": "Bill",
"date": "2020-03-11T20:02:38.000Z"
},
{
"name": "Steve",
"date": "2020-03-19T20:02:38.000Z"
}
]
}
As you can see the it is not sorted by date - desc order. The result should be like shown below
"userdata": [
{
"name": "Elon",
"date": "2020-03-21T20:02:38.143Z"
},
{
"name": "Steve",
"date": "2020-03-19T20:02:38.000Z"
},
{
"name": "Bill",
"date": "2020-03-11T20:02:38.000Z"
}
]
$sort will use the last object which comes out from the aggregate pipe and theres no field "date" in this object:
{
$group: {
_id: "$matchedTask._id",
name: {$first: "$matchedTask.name"},
icon: {$first: "$matchedTask.icon"},
assignee: { $first: "$matchedTask.assignee" },
userdata: {
$push: {
name: "$matchedUser.name",
execDate: "$date"
}
}
}
},
your group has to returned a field named date in order to be able to sort it
{
$group: {
_id: "$matchedTask._id",
name: ....,
date: ....
}
}
{ $sort: {date: -1}}
if the value you want to sort is indide another object you must specify it on sort:
{$sort: {"userdata.date": -1}}
I fixed it, had to use sort two times, now I am able to get the result as I want
Solution provided below
UserTask.aggregate([
{
$lookup: {
from: "tasks",
localField: "task",
foreignField: "_id",
as: "matchedTask"
}
},
{
$unwind: "$matchedTask"
},
{
$lookup: {
from: "users",
localField: "user",
foreignField: "_id",
as: "matchedUser"
}
},
{
$unwind: "$matchedUser"
},
{
$sort: { date: -1 }
},
{
$group: {
_id: "$matchedTask._id",
name: { $first: "$matchedTask.name" },
icon: { $first: "$matchedTask.icon" },
assignee: { $first: "$matchedTask.assignee" },
userdata: {
$push: {
name: "$matchedUser.name",
execDate: "$date"
}
}
}
},
{
$sort: { _id: 1 }
}
])
.exec()
.then(doc => res.status(200).json(doc))
.catch(err => res.status(400).json("Error: " + err));
I have the following document
{
"userid": "5a88389c9108bf1c48a1a6a7",
"email": "abc#gmail.com",
"lastName": "abc",
"firstName": "xyz",
"__v": 0,
"friends": [{
"userid": "5a88398b9108bf1c48a1a6a9",
"ftype": "SR",
"status": "ACCEPT",
"_id": ObjectId("5a9585b401ef0033cc8850c7")
},
{
"userid": "5a88398b9108bf1c48a1a6a91111",
"ftype": "SR",
"status": "ACCEPT",
"_id": ObjectId("5a9585b401ef0033cc8850c71111")
},
{
"userid": "5a8ae0a20df6c13dd81256e0",
"ftype": "SR",
"status": "pending",
"_id": ObjectId("5a9641fbbc9ef809b0f7cb4e")
}]
},
{
"userid": "5a88398b9108bf1c48a1a6a9",
"friends": [{ }],
"lastName": "123",
"firstName": "xyz",
.......
},
{
"userid": "5a88398b9108bf1c48a1a6a91111",
"friends": [{ }],
"lastName": "456",
"firstName": "xyz",
...
}
First Query
Here I want to get userId from friends array ,which having status equals to "ACCEPT".
ie
[5a88398b9108bf1c48a1a6a9,5a88398b9108bf1c48a1a6a91111]
Second Query
After that, I have to make another query on the same collection to get details of each userid returned in the first query.
final Query will return details of [5a88398b9108bf1c48a1a6a9,5a88398b9108bf1c48a1a6a91111]
both userid ie
[
{
userid" : "5a88398b9108bf1c48a1a6a9",
"lastName" : "123",
"firstName" : "xyz"
},
{
"userid" : "5a88398b9108bf1c48a1a6a91111",
"lastName" : "456",
"firstName" : "xyz"
}
]
I have tried so far with
Users.find ({'_id':5a88389c9108bf1c48a1a6a7,"friends.status":'ACCEPT'}, (error, users) => {})
or
Users.find ({'_id':5a88389c9108bf1c48a1a6a7, friends: { $elemMatch: { status: 'ACCEPT' } } }, (error, users) => {})
Use the aggregation framework's $map and $filter operators to handle the task. $filter will filter the friends array based on the specified condition that the status should equal "ACCESS" and $map will transform the results from the filtered array to the desired format.
For the second query, append a $lookup pipeline step which does a self-join on the users collection to retrieve the documents which match the ids from the previous pipeline.
Running the following aggregate operation will produce the desired array:
User.aggregate([
{ "$match": { "friends.status": "ACCEPT" } },
{ "$project": {
"users": {
"$map": {
"input": {
"$filter": {
"input": "$friends",
"as": "el",
"cond": { "$eq": ["$$el.status", "ACCEPT"] }
}
},
"as": "item",
"in": "$$item.userid"
}
}
} },
{ "$lookup": {
"from": "users",
"as": "users",
"localField": "users",
"foreignField": "userid"
} },
]).exec((err, results) => {
if (err) throw err;
console.log(results[0].users);
});
I did not test it. just for an idea, give it a try and let me know.
db.Users.aggregate(
[
{
$unwind: "$friends"
},
{
$match:{ "$friends.status": "ACCEPT"}
},
{
$project:{ "FriendUserID":"$friends.userid"}
},
{
$lookup:{
from:"Users",
as: "FriendsUsers",
localField: "FriendUserID",
foreignField: "userid"
}
},
{
$project: { FriendsUsers.lastName:1,FriendsUsers.firstName:1 }
}
]
)
filtering nested elements
const products = await Product.aggregate<ProductDoc>([
{
$match: {
userId: data.id,
},
},
{
$project: {
promotions: {
$filter: {
input: '$promotions',
as: 'p',
cond: {
$eq: ['$$p.status', PromotionStatus.Started],
},
},
},
userId: 1,
name: 1,
thumbnail: 1,
},
},
]);
for multiple condition
cond: {
$and: [
{
$eq: [
"$$c.product",
"37sd87hjsdj3"
]
},
{
$eq: [
"$$c.date",
"date-jan-4-2022"
],
}
]
},