Join MongoDB subdocuments in same parent document - node.js

Given this Orders collection:
// Order documents
[
{
_id: "order_123",
items: [
{ _id: "item_123", type: "T-Shirt" },
{ _id: "item_234", type: "Hoodie" },
{ _id: "item_345", type: "Hat" },
],
refunds: [
{
_id: "refund_123",
items: ["item_123", "item_234"],
},
{
_id: "refund_234",
items: ["item_345"],
},
],
},
]
Is it possible to map refunds.items -> items._id, allowing us to filter by type?
This is how we currently get the refund sub-documents:
db.orders.aggregate([
{
$replaceRoot: {
newRoot: {
order: "$$ROOT",
refunds: "$$ROOT.refunds",
},
},
},
{
$unwind: "$refunds",
},
{
$project: {
order: "$order",
refund: "$refunds",
},
},
]);
Which gives us:
// Refund documents
[
{
refund: {
_id: "refund_123",
items: ["item_123", "item_234"],
},
order: { ... }, // The original order document
},
{
refund: {
_id: "refund_234",
items: ["item_345"],
},
order: { ... }, // The original order document
},
]
From here, we want to map up refund.items -> order.items._id to produce the following output:
[
{
_id: "refund_123",
items: [
{ _id: "item_123", type: "T-Shirt" },
{ _id: "item_234", type: "Hoodie" },
],
},
{
_id: "refund_234",
items: [
{ _id: "item_345", type: "Hat" }
],
},
]
Allowing us to filter refund documents by type.

You can do this using $unwind and $filter,
$unwind deconstruct array refunds
$project to show refund id in _id, and filter items that are in refunds.items array using $filter
db.orders.aggregate([
{ $unwind: "$refunds" },
{
$project: {
_id: "$refunds._id",
items: {
$filter: {
input: "$items",
cond: { $in: ["$$this._id", "$refunds.items"] }
}
}
}
}
])
Playground

Related

Mongodb $lookup inside $addfield?

I have a collection named users, and this is how one specific user will look like:
{
_id: 'Object ID',
name: 'String',
cart: [
{
product_id: 'Product object ID',
quantity: 'Number',
},
...
],
}
I want my desired results to look like this:
{
_id: 'Object ID',
name: 'String',
cart: [
{
product_id: 'Product object ID',
quantity: 'Number',
product_details: {
'all the details of the product from Products collection which matches the product_id',
},
},
...
],
}
I tried adding addFields into lookup but it's getting too complicated and doesn't work as desired. What's the best way to aggregate this?
You can achieve this in several different ways, here's what I consider to be the most simple:
db.users.aggregate([
{
"$lookup": {
"from": "products",
let: {
cart: "$cart"
},
pipeline: [
{
$match: {
$expr: {
$in: [
"$_id",
"$$cart.product_id"
]
}
}
},
{
$replaceRoot: {
newRoot: {
"$mergeObjects": [
"$$ROOT",
{
"$arrayElemAt": [
{
$filter: {
input: "$$cart",
cond: {
$eq: [
"$_id",
"$$this.product_id"
]
}
}
},
0
]
}
]
}
}
}
],
"as": "cart"
}
}
])
Mongo Playground

Group $lookup results within inner nested array, and keep outer nested array and its contents intact in MongoDB

I have a collection which comprises of three level array nesting as shown below
_id: ObjectID('abc'),
sections: [
{
sectionId: "sec0",
sectionName: "ABC",
contents: [
{
contentId: 0,
tasks: [
{
taskId: ObjectID('task1')
}
//May contain 1-100 tasks
],
contentDescription: "Content is etc",
}
]
}
]
Sections is an array of objects which contains an object each with sectionId, and contents array which is an array of objects comprising of contentId, contentDescription, and nested array of tasks which comprises of an object containing a taskId.
I am applying $lookup operator in order to join nested tasks array with tasks collection but I am facing a problem in document duplication as shown below.
_id: ObjectID('abc'),
sections: [
{
sectionId: "sec0",
sectionName: "ABC",
contents: [
{
contentId: 0,
tasks: [
{
//Task Document of ID 1
}
],
contentDescription: "Content is etc",
}
]
}
]
_id: ObjectID('abc'),
sections: [
{
sectionId: "sec0",
sectionName: "ABC",
contents: [
{
contentId: 0,
tasks: [
{
//Task Document of ID 2
}
],
contentDescription: "Content is etc",
}
]
}
]
Whereas the desired output is as follows
_id: ObjectID('abc'),
sections: [
{
sectionId: "sec0",
sectionName: "ABC",
contents: [
{
contentId: 0,
tasks: [
{
//Task Document of ID 1
},
{
//Task Document of ID 2
},
{
//Task Document of ID 3
}
],
contentDescription: "Content is etc",
}
]
}
]
In the collection, a sections array might contain multiple section object which might contain multiple contents and so on and so forth.
The schema in question is temporary as our company is currently migrating from an existing database to MongoDB, so architectural refactoring is not possible atm and I need to work with existing schema design from different database.
I tried the following way
const contents= await sections.aggregate([
{
$match: { _id: id},
},
{ $unwind: '$sections' },
{
$unwind: {
path: '$sections.contents',
preserveNullAndEmptyArrays: true,
},
},
{
$unwind: {
path: '$sections.contents.tasks',
preserveNullAndEmptyArrays: true,
},
},
{
$lookup: {
from: 'tasks',
let: { task_id: '$sections.contents.tasks.taskId' },
pipeline: [
{ $match: { $expr: { $eq: ['$_id', '$$task_id'] } } },
],
as: 'sections.contents.tasks',
},
},
{
$addFields: {
'sections.contents.tasks': {
$arrayElemAt: ['$sections.contents.tasks', 0],
},
},
},
{
$group: {
_id: '$_id',
exam: { $push: '$sections.contents.tasks' },
},
},
]);
And I am also unable to use $group aggregation operator like
$group: {
_id: '$_id',
sections: {
sectionId : { $first: '$sectionId' },
sectionName: { $first: '$sectionName' },
contents: {
contentId: { $first: '$contentId' },
task: { $push: $sections.contents.tasks }
}
},
},
Any help or directions will be appreciated, I also searched on SO, and found this but couldn't understand the following part
{"$group":{
"_id":{"_id":"$_id","mission_id":"$missions._id"},
"agent":{"$first":"$agent"},
"title":{"$first":"$missions.title"},
"clients":{"$push":"$missions.clients"}
}},
{"$group":{
"_id":"$_id._id",
"missions":{
"$push":{
"_id":"$_id.mission_id",
"title":"$title",
"clients":"$clients"
}
}
}}
So you're very close to the final solution, a good "rule" that's good to remember is if you unwind x times you need to group x to restore the original structure properly, like so:
db.collection.aggregate([
{
$match: {
_id: id
},
},
{
$unwind: "$sections"
},
{
$unwind: {
path: "$sections.contents",
preserveNullAndEmptyArrays: true,
},
},
{
$unwind: {
path: "$sections.contents.tasks",
preserveNullAndEmptyArrays: true,
},
},
{
$lookup: {
from: "tasks",
let: {
task_id: "$sections.contents.tasks.taskId"
},
pipeline: [
{
$match: {
$expr: {
$eq: [
"$_id",
"$$task_id"
]
}
}
},
],
as: "sections.contents.tasks",
},
},
{
$addFields: {
"sections.contents.tasks": {
$arrayElemAt: [
"$sections.contents.tasks",
0
],
},
},
},
{
$group: {
_id: {
contentId: "$sections.contents.contentId",
sectionId: "$sections.sectionId",
sectionName: "$sections.sectionName",
originalId: "$_id"
},
tasks: {
$push: "$sections.contents.tasks"
},
contentDescription: {
$first: "$sections.contents.contentDescription"
},
}
},
{
$group: {
_id: {
sectionId: "$_id.sectionId",
sectionName: "$_id.sectionName",
originalId: "$_id.originalId"
},
contents: {
$push: {
contentId: "$_id.contentId",
tasks: "$tasks",
contentDescription: "$contentDescription"
}
}
}
},
{
$group: {
_id: "$_id.originalId",
sections: {
$push: {
sectionId: "$_id.sectionId",
sectionName: "$_id.sectionName",
contents: "$contents"
}
}
}
}
])
Mongo Playground
However your pipeline could be made a little cleaner as it has 1 redundant $unwind stage that also adds a redundant $group stage. I won't post the entire fixed pipeline here as it's already a long post but feel free to check it out here: Mongo Playground fixed

search from child collection contains the keyword from parent and child collection

I am facing a problem with getting data from database using parent child relationship collections.
Here is my collection structure --
-post
---post cloth : brand id from brands collections
-----brand
Now I am getting data from post and post cloth with keyword search from post cloth and brand table if any key matches from post cloth and brand. Till post cloth it is working fine along with keyword search in or condition, Now I also need to search from brand and return the result if keyword contains in brand as well.
here are my cases --
data returned : if any of post_cloths keys matches the keyword searched
data returned : if any of the post_cloths keys matches the keyword OR lookup with brand name matches the keyword
data returned : if all keys from post_cloths not matches the keyword but lookup with brand name matches the keyword
data not returned : if no keys from post_cloths matches the keyword and also lookup with brand name not matches the keyword
Here is my code :
var page = 0;
if (req.query.page >= 0) {
page = req.query.page;
}
let filter = { 'totalCloth': { $gte: 1 } };
if (req.query.user != null && req.query.user != '') {
filter.createdBy = ObjectID(req.query.user);
}
console.log(filter);
var searchQuery = [];
var brandSearchQuery = [];
if (req.query.keyword != null && req.query.keyword != '') {
console.log(req.query.keyword);
keyword = req.query.keyword;
searchQuery = [
{
$regexFind: {
input: '$category',
regex: new RegExp(keyword),
options: 'i',
},
},
{
$regexFind: {
input: '$color',
regex: new RegExp(keyword),
options: 'i',
},
},
{
$regexFind: {
input: '$country',
regex: new RegExp(keyword),
options: 'i',
},
},
{
$regexFind: {
input: '$size',
regex: new RegExp(keyword),
options: 'i',
},
},
{
$regexFind: {
input: '$clothMaterial',
regex: new RegExp(keyword),
options: 'i',
},
},
];
brandSearchQuery = [
{
$regexFind: {
input: '$name',
regex: new RegExp(keyword),
options: 'i',
},
},
];
} else {
searchQuery = [{}];
brandSearchQuery = [{}];
}
// get the post details
// PostModel.find(filter).countDocuments().then(countPosts => {
PostModel.aggregate([
{
$lookup: {
from: 'post_cloths',
let: { postId: '$_id' },
pipeline: [
//lookup for brand
{
$lookup: {
from: 'brands',
let: { brandId: '$brandId' },
pipeline: [
{
$match: {
$expr:
{
$and:
[
{ $eq: ['$_id', '$$brandId'] },
{ $or: brandSearchQuery },
],
},
},
},
],
as: 'brand',
},
},
//end of brand lookup
{
$match: {
$expr: {
$and:
[
{ $eq: ['$postId', '$$postId'] },
{
$or: searchQuery,
},
],
},
},
},
{
$project: {
totalBrands: { $size: '$brand' },
},
},
{
$match: {
$expr:
{ $or: [{ $match: { totalBrands: { $gte: 1 } } }] },
},
},
],
as: 'postCloth',
},
},
{
$project: {
image: 1,
createdAt: 1,
createdBy: 1,
mediaUrl: {
$concat: [process.env.PROJECT_URL + '/files/', '$image'],
},
totalCloth: { $size: '$postCloth' },
},
},
//check for post cloth object if length is greater than equals to 1
{
$match: filter,
},
{ $skip: 12 * page },
{ $limit: 12 },
{ $sort: { createdAt: -1 } },
]).exec(function(err, post) {
return apiResponse.successResponseWithData(res, 'Successful', post);
});
I am getting data properly, but not while searching from brand. Please suggest how we can search the data from the cases given. there is simple keyword search.
Thanks in advance
The problem is with you're main's $lookup's pipeline:
first you start with the brand $lookup, which i'll assume works ( if you provide schema's for your collections it would be easy to verify), however right after that $lookup you do this:
{
$match: {
$expr:
{
$and:
[
{ $eq: ['$postId', '$$postId'] },
{
$or: searchQuery,
},
],
},
},
},
This means if the searchQuery fails even if a brand exists the document will be filtered out, you should change it to:
{
$match: {
$expr:
{
$and:
[
{ $eq: ['$postId', '$$postId'] },
{
$or: [
{
$or: searchQuery
},
{
$gt: [{$size: "$brand"}, 0]
}
],
},
],
},
},
},
Now this will also matched documents that have any brands in the brand field, meaning the brand matched the nested $lookup, you can then drop the next 2 stages that check for the brand size.
I also recommend that you move the $eq for the postId to the start of the $lookup, this will improve performance immensely, after all the changes the entire pipeline would look like:
PostModel.aggregate([
{
$lookup: {
from: 'post_cloths',
let: { postId: '$_id' },
pipeline: [
{
$match: { $eq: ['$postId', '$$postId'] },
},
{
$lookup: {
from: 'brands',
let: { brandId: '$brandId' },
pipeline: [
{
$match: {
$expr:
{
$and:
[
{ $eq: ['$_id', '$$brandId'] },
{ $or: brandSearchQuery },
],
},
},
},
],
as: 'brand',
},
},
{
$match: {
$expr: {
$and:
[
{
$or: [
{
$or: searchQuery,
},
{
$gt: [{ $size: '$brand' }, 0],
},
],
},
],
},
},
},
],
as: 'postCloth',
},
},
{
$project: {
image: 1,
createdAt: 1,
createdBy: 1,
mediaUrl: {
$concat: [process.env.PROJECT_URL + '/files/', '$image'],
},
totalCloth: { $size: '$postCloth' },
},
},
{
$match: filter,
},
{ $skip: 12 * page },
{ $limit: 12 },
{ $sort: { createdAt: -1 } },
])

Group nested fields with aggregation and return results with others fields mongo db

I need to change the structure of some field in my mongoDB document.
Here the sample:
[
{
_id: "ObjectId('997v2ha1cv9b0036fa648zx3')",
title: "Adidas Predator",
size: "8",
colors: [
{
hex: "005FFF",
name: "Blue"
},
{
hex: "FF003A",
name: "Red"
},
{
hex: "FFFE00",
name: "Yellow"
},
{
hex: "07FF00",
name: "Green"
},
],
extras: [
{
description: "laces",
type: "exterior"
},
{
description: "sole",
type: "interior"
},
{
description: "logo"
},
{
description: "stud",
type: "exterior"
}
],
media: {
images: [
{
url: "http://link.com",
type: "exterior"
},
{
url: "http://link3.com",
type: "interior"
},
{
url: "http://link2.com",
type: "interior"
},
{
url: "http://link4.com",
type: "exterior"
}
]
}
}
];
My goal is to group some fields:
colors need to be and array with just the colors,
extras need to be an array with 3 object each one for a "type" (interior, exterior, null)
the same for images that is inside media
Here what I expected:
{
_id: "ObjectId('997b5aa1cv9b0036fa648ab5')",
title: "Adidas Predator",
size: "8",
colors: ["Blue", "Red", "Yellow", "Green"],
extras: [
{type: exterior, description: ["laces", "stud"]},
{type: interior, description: ["sole"]},
{type: null, description: ["logo"]}
],
images: [
{type: exterior, url: ["http://link.com", "http://link4.com"]},
{type: interior, url: ["http://link2.com", "http://link3.com"]},
]
};
With my code I can achieve my goal but I don't understand how to show all the information together through the pipeline.
Here my code:
db.collection.aggregate([
{
$project: {
title: 1,
size: 1,
colors: "$colors.name",
extras: 1,
media: "$media.images"
},
},
{
$unwind: "$media"
},
{
$group: {
_id: {
type: "$media.type",
url: "$media.url",
},
},
},
{
$group: {
_id: "$_id.type",
url: {
$push: "$_id.url"
},
},
},
]);
The result is:
[
{
_id: "exterior",
url: [
"http://link.com",
"http://link4.com"
]
},
{
_id: "interior",
url: [
"http://link3.com",
"http://link2.com"
]
}
];
If I do the same thing with extras I get the same (correct) structure.
How can I show all the data together like in the expected structure?
Thanks in advice.
The strategy will be to maintain the require parent fields throughout the pipeline using $first to just grab the initial value, It ain't pretty but it works:
db.collection.aggregate([
{
"$addFields": {
colors: {
$map: {
input: "$colors",
as: "color",
in: "$$color.name"
}
}
}
},
{
$unwind: "$extras"
},
{
"$addFields": {
imageUrls: {
$map: {
input: {
$filter: {
input: "$media.images",
as: "image",
cond: {
$eq: [
"$$image.type",
"$extras.type"
]
}
}
},
as: "image",
in: "$$image.url"
}
}
}
},
{
$group: {
_id: {
_id: "$_id",
extraType: "$extras.type"
},
extraDescriptions: {
"$addToSet": "$extras.description"
},
imageUrls: {
"$first": "$imageUrls"
},
colors: {
$first: "$colors"
},
size: {
$first: "$size"
},
title: {
$first: "$title"
}
}
},
{
$group: {
_id: "$_id._id",
colors: {
$first: "$colors"
},
size: {
$first: "$size"
},
title: {
$first: "$title"
},
images: {
$push: {
type: {
"$ifNull": [
"$_id.extraType",
null
]
},
url: "$imageUrls"
}
},
extras: {
$push: {
type: {
"$ifNull": [
"$_id.extraType",
null
]
},
description: "$extraDescriptions"
}
}
}
}
])
Mongo Playground
You can try $function operator, to defines a custom aggregation function or expression in JavaScript.
$project to show required fields and get array of colors name
$function, write your JS logic if you needed you can sort this logic of group, it will return result with 2 fields (extras, images)
$project to show required fields and separate extras and images field from result
db.collection.aggregate([
{
$project: {
title: 1,
size: 1,
colors: "$colors.name",
result: {
$function: {
body: function(extras, images) {
function groupBy(objectArray, k, v) {
var results = [], res = objectArray.reduce((acc, obj) => {
if (!acc[obj[k]]) acc[obj[k]] = [];
acc[obj[k]].push(obj[v]);
return acc;
}, {});
for (var o in res) {
results.push({ [k]: o === 'undefined' ? null : o, [v]: res[o] })
}
return results;
}
return {
extras: groupBy(extras, 'type', 'description'),
images: groupBy(images, 'type', 'url')
}
},
args: ["$extras", "$media.images"],
lang: "js"
}
}
}
},
{
$project: {
title: 1,
size: 1,
colors: 1,
extras: "$result.extras",
images: "$result.images"
}
}
])
Playground
IMPORTANT:
Executing JavaScript inside an aggregation expression may decrease performance. Only use the $function operator if the provided pipeline operators cannot fulfill your application's needs.

I need mongodb group pipeline for multiple condition

Document
[
{
type: 1,//credit
amount: 60
},
{
type: 2,//debit
amount: 35
},
{
type: 3,//credit
amount: 25
},
{
type: 4,//debit
amount: 80
},
{
type: 5,//credit
amount: 70
},
]
Result
[
{
_id: {
Name: "Credition",
Type: [1, 3, 5]
},
Total_Amount: 155
},
{
_id: {
Name: "Debition",
Type: [2, 4]
},
Total_Amount: 115
},
]
In my schema, there are millions of logs records in which few are credited logs, few are debited logs.
I want to use MongoDB aggregate pipe and have to group like above for million records at a time
Yes you can do that first you need to add a new field transaction on the basis of the type of logs, then you can group the logs on the basis of that field.
Working example - https://mongoplayground.net/p/e4kqeKLIuIr
db.collection.aggregate([
{
$addFields: {
transaction: {
$cond: {
if: {
$in: [
"$type",
[
1,
3,
5
]
]
},
then: "Credition",
else: "Debition"
}
}
}
},
{
$group: {
_id: "$transaction",
Type: {
$addToSet: "$type"
},
Total_Amount: {
$sum: "$amount"
}
}
}
])
After this, you can also use $project operator to change the name or structure of the record, if needed
You can use the operator $cond during the grouping stage:
db.collection.aggregate([
{
$group: {
_id: {
$cond: [
{
$in: [ "$type", [1,3,5] ]
},
"Credition",
"Debition"
]
},
type: {
$addToSet: "$type"
},
amount: {
$sum: "$amount"
}
}
},
{
$project: {
_id: {
Name: "$_id",
Type: "$type"
},
Total_Amount: "$amount"
}
}
])
MongoPlayground

Resources