I have two collections (promotions,product) and product collection map to promotions its working fine.but I have doubt how to show particular columns in product collection.
promotional collection
{
"_id" : ObjectId("5cf7679a0b0bed2e7483b998"),
"group_name" : "Latest",
"products" :
[ObjectId("5cecc161e8c1e73478956333"),ObjectId("5cecc161e8c1e73478956334")]
}
product collection
{
"_id" : ObjectId("5cecc161e8c1e73478956333"),
"product_name" : "bourbon"
},
{
"_id" : ObjectId("5cecc161e8c1e73478956334"),
"product_name" : "bour"
}
mapping query
db.promotional.aggregate(
[
{
$lookup: {
from: "product",
localField: "products",
foreignField: "_id",
as: "products"
}
}
]
)
I tried to map product collection to promotional collection
I got Output
{
"_id" : ObjectId("5cf7679a0b0bed2e7483b998"),
"group_name" : "Latest",
"products" :
[
{
"_id" : ObjectId("5cecc161e8c1e73478956333"),
"product_name" : "bourbon"
},
{
"_id" : ObjectId("5cecc161e8c1e73478956334"),
"product_name" : "bour"
}
]
}
Expected output
{
"_id" : ObjectId("5cf7679a0b0bed2e7483b998"),
"group_name" : "Latest",
"products" :
[
{
"product_name" : "bourbon"
},
{
"product_name" : "bour"
}
]
}
You can exclude those columns using $project operator:
db.promotional.aggregate(
[
{
$lookup: {
from: "product",
localField: "products",
foreignField: "_id",
as: "products"
}
},
{
$project: {
"products._id": 0
}
}
]
)
db.promotional.aggregate([
{
$lookup: {
from: "product",
localField: "products",
foreignField: "_id",
as: "products"
}
},{$project :{products :{product_name : 1}}}
])
db.getCollection("promotional").aggregate(
// Pipeline
[
// Stage 1
{
$unwind: {
path: "$products"
}
},
// Stage 2
{
$lookup: {
from: "product",
localField: "products",
foreignField: "_id",
as: "products"
}
},
// Stage 3
{
$group: {
_id: {
_id: '$_id',
group_name: '$group_name'
},
products: {
$push: {
product_name: {
$arrayElemAt: ["$products.product_name", 0]
}
}
}
}
},
// Stage 4
{
$project: {
_id: '$_id._id',
group_name: '$_id.group_name',
products: 1
}
},
]
);
Related
I am working with node - js and mongoDB , all the queries are ok and working fine but at some point I am using a query used 4 lookups , also applied the matches based on those lookups , and applied the pagination+ sorting in the same query. But the main issue I am facing is the query taking time around 10-20 seconds to fetch the data from the database , which is really too long time period.
Here is the code snippet for the same
var products = await db.collection('catalog_products')
.aggregate([
{ $match: { categories: cat_id.toString(), status:1, verification_status:1}},
{ $lookup: { from: 'catalog_product_meta', localField: '_id', foreignField: 'product_id', as: 'meta' } }, { $unwind:"$meta" },
{ $lookup: { from: 'catalog_product_attributes', localField: '_id', foreignField: 'product_id', as: 'attributes' } }, { $unwind:"$attributes" },
{$match : {$and : [ { $or : [ { "attributes.attribute_value" : ObjectId("60f5626681cc91c83a34f6c8") }, { "attributes.attribute_value" : ObjectId("617285baaad0c6b9d269a6c5") } ] }, { $or : [ { "attributes.attribute_value" : ObjectId("61600701dc103aaf206165c3") } ] } ]}},
{ $lookup: { from: 'catalog_product_prices', localField: '_id', foreignField: 'product_id', as: 'prices' } }, { $unwind:"$prices" },
{ $match : {$and:{ 'prices.regular_price': { '$gte': 3109, '$lte': 15406 } }}},
{ $sort: sort},
{ $project: {
"_id" : 1,
"name" : "$meta.name",
"url_key" : 1,
"regular_price" : "$prices.regular_price",
"sale_price" : "$prices.sale_price",
} },
],{ "allowDiskUse" : true }).skip(36).limit(36).toArray();
I am trying to find documents in a collection, but filtered based on the value of an embedded ObjectID relation.
Mongoose schema is as follows:
const UserQualificationSchema = new Schema(
{
user: {
type: mongoose.Schema.Types.ObjectId,
ref: 'User',
},
}
const UserSchema = new Schema(
{
fleet: {
type: [String], // Eg ["FleetA", "FleetB", "FleetC"]
required: true,
}
}
I need to find all UserQualifications where an item in the user's fleet equals a value in a filter array.
For example: Find all User Qualifications where user.fleet: {$in: ["FleetA", "FleetC"]}
I've looked at aggregations and querying inside .populate() but can't seem to get it to work.
Any ideas much appreciated.
Use aggregation query for your problem, I have created a query for you.
users.collection.json
/* 1 */
{
"_id" : ObjectId("61056c4a8cca27df3db2e4c8"),
"firstName" : "Rahul",
"lastName" : "soni",
"fleet" : [
"FleetA",
"FleetB",
"FleetC"
],
"createdAt" : ISODate("2021-07-31T15:29:14.918Z")
}
userqualifications.collection.json
/* 1 */
{
"_id" : ObjectId("61056c908cca27df3db2e4c9"),
"user" : ObjectId("61056c4a8cca27df3db2e4c8"),
"createdAt" : ISODate("2021-07-31T15:30:24.510Z")
}
aggregation query:
it will get the result only if a user has FleetA and FleetC.
if anyone is not matched then it will return 0 records
db.userqualifications.aggregate([{
"$lookup": {
"from": "users",
"localField": "user",
"foreignField": "_id",
"as": "user"
}
}, {
"$unwind": "$user"
}, {
"$match": {
"user.fleet": {
"$elemMatch": {
"$eq": "FleetA",
"$eq": "FleetC"
}
}
}
}])
Result:
/* 1 */
{
"_id" : ObjectId("61056c908cca27df3db2e4c9"),
"user" : {
"_id" : ObjectId("61056c4a8cca27df3db2e4c8"),
"firstName" : "Rahul",
"lastName" : "soni",
"fleet" : [
"FleetA",
"FleetB",
"FleetC"
],
"createdAt" : ISODate("2021-07-31T15:29:14.918Z")
},
"createdAt" : ISODate("2021-07-31T15:30:24.510Z")
}
if the goal is to get only UserQualifications then the following should be an efficient way as it can use an index on the fleet field of the User collection.
db.User.aggregate([
{
$match: {
fleet: { $in: ["FleetA", "FleetB"] }
}
},
{
$lookup: {
from: "UserQualification",
localField: "_id",
foreignField: "user",
as: "qualifications"
}
},
{
$unwind: "$qualifications"
},
{
$replaceWith: "$qualifications"
}
])
on the other hand if you start from the UserQualifications collection, you can't efficiently narrow down the records as you're filtering on something that it doesn't have the data for.
Thank you for the answer - it did achieve the results I was looking for - however I am now struggling to add a $match with $and to the aggregate to only return the qualifications where the user ID equals one inside a submitted array AND a given fleet.
I have the following aggregate:
db.UserQualifications.aggregate([{
{
$lookup: {
from: 'users',
localField: 'user',
foreignField: '_id',
as: 'user',
},
},
{
$unwind: '$user',
},
{
$match: {
$and: [
'user.fleet': {
$in: ["Fleet A", "Fleet C"], // This works on it's own
},
user: { // Also tried 'user._id'
$in: ["6033e4129070031c07fbbf29"] // Adding this returns blank array
}
]
},
}
}])
I have double checked that I am passing in the correct User ID's inside the arrays, but when I add this to the $and inside match, it only returns a blank array.
Is there another way to do this?
// Updated users collection
/* 1 */
{
"_id" : ObjectId("61056c4a8cca27df3db2e4c8"),
"firstName" : "Rahul",
"lastName" : "soni",
"fleet" : [
"Fleet A",
"Fleet B",
"Fleet C"
],
"createdAt" : ISODate("2021-07-31T15:29:14.918Z")
}
Query:
// userqualifications => this is my collection name, you can add your collection name here db.<YOUR>
db.userqualifications.aggregate([{
$lookup: {
from: 'users',
localField: 'user',
foreignField: '_id',
as: 'user',
},
},
{
$unwind: '$user',
},
{
$match: {
// $and operatory syntax [{}, {}]
$and: [{
'user.fleet': {
// "Fleet A", "Fleet C" (FleetA, FleetC) this is the my first options,
// I have changes according to your problem
$in: ["Fleet A", "Fleet C"], // This works on it's own
}
}, {
// Convert user id to ObjectId type (_bsonType)
"user._id": ObjectId("61056c4a8cca27df3db2e4c8")
}]
}
}
])
Result:
/* 1 */
{
"_id" : ObjectId("61056c908cca27df3db2e4c9"),
"user" : {
"_id" : ObjectId("61056c4a8cca27df3db2e4c8"),
"firstName" : "Rahul",
"lastName" : "soni",
"fleet" : [
"Fleet A",
"Fleet B",
"Fleet C"
],
"createdAt" : ISODate("2021-07-31T15:29:14.918Z")
},
"createdAt" : ISODate("2021-07-31T15:30:24.510Z")
}
Difference Image:
I have 3 mongoDB collections
I need to aggregate them with $lookup operator but I didn't find anything/**or I'm bad looking **
1st one is suppliers
{
"_id" : ObjectId("111"), //for example, in db is mongodb ids
"name" : "supplier 1",
}
{
"_id" : ObjectId("222"),
"name" : "supplier 1",
}
2nd one is clients
{
"_id" : ObjectId("333"), //for example, in db is mongodb ids
"name" : "clients 1",
}
{
"_id" : ObjectId("444"),
"name" : "clients 2",
}
and 3rd is moves
{
"_id" : ObjectId("..."), //for example, in db is mongodb ids
"moveName" : "move 1",
"agent": ObjectId("111") // this is from suppliers collection
}
{
"_id" : ObjectId("..."),
"moveName" : "move 2",
"agent": ObjectId("333") // this one is from CLIENTS collection
}
so like output I need data like this
{
"_id" : ObjectId("..."), //for example, in db is mongodb ids
"moveName" : "move 1",
**"agent": supplier 1** // this is from suppliers collection
}
{
"_id" : ObjectId("..."),
"moveName" : "move 2",
**"agent": clients 1** // this one is from CLIENTS collection
}
back end is nodejs, I`m using mongoose, how I can search in 2nd collection if noresult in 1st?
const moves = await Move.aggregate([
{ $match: query }, // here all wokrs good
{
$lookup: {
from: 'clients',
localField: 'agent',
foreignField: '_id',
as: 'agent'
}
},{ $unwind: {path: "$agent" , preserveNullAndEmptyArrays: true} },
{
$lookup: {
from: 'suppliers',
localField: 'agent',
foreignField: '_id',
as: 'agent2'
}
},
{
$project: {
operationName: 1,
agent: {$ifNull: ['$agent.name', '$agent2.name']}
}
}
])
Thank You!
As suggested by #hhharsha36, we can use $facet operator which allows to run several pipelines within a single stage.
Explanation
facet
suppliers = $lookup suppliers collection and filter only matched results
clientes = $lookup clientes collection and filter only matched results
concatArrays = We concat suppliers and clients results into a single movies array
unwind = We flatten movies array [a, b, c] -> a
b
c
replaceWith = We replace the root element [movies:a, movies:b -> a, b]
mergeObject = allows us to pick the agent name (this way we avoid 1 more stage)
db.moves.aggregate([
{
$facet: {
suppliers: [
{
$lookup: {
from: "suppliers",
localField: "agent",
foreignField: "_id",
as: "agent"
}
},
{
$match: {
agent: {
$not: {
$size: 0
}
}
}
}
],
clients: [
{
$lookup: {
from: "clients",
localField: "agent",
foreignField: "_id",
as: "agent"
}
},
{
$match: {
agent: {
$not: {
$size: 0
}
}
}
}
]
}
},
{
$project: {
movies: {
"$concatArrays": [
"$clients",
"$suppliers"
]
}
}
},
{
$unwind: "$movies"
},
{
$replaceWith: {
"$mergeObjects": [
"$movies",
{
agent: {
"$arrayElemAt": [
"$movies.agent.name",
0
]
}
}
]
}
}
])
MongoPlayground
This aggregation query gives the desired result:
db.moves.aggregate([
{
$lookup: {
from: "suppliers",
localField: "agent",
foreignField: "_id",
as: "moves_sup"
}
},
{
$unwind: { path: "$moves_sup" , preserveNullAndEmptyArrays: true }
},
{
$lookup: {
from: "clients",
localField: "agent",
foreignField: "_id",
as: "moves_client"
}
},
{
$unwind: { path: "$moves_client" , preserveNullAndEmptyArrays: true }
},
{
$addFields: {
agent: {
$cond: [ { $eq: [ { $type: "$moves_sup" }, "object" ] },
"$moves_sup.name",
{ $cond: [ { $eq: [ { $type: "$moves_client" }, "object" ] }, "$moves_client.name", "undefined" ] }
] },
moves_client: "$$REMOVE",
moves_sup: "$$REMOVE"
}
},
])
Now consider the case , i have one document containing below collection like structure.
Below is the order collection
{
"_id" : ObjectId("5788fcd1d8159c2366dd5d93"),
"color" : "Blue",
"code" : "1",
"category_id" : ObjectId("5693d170a2191f9020b8c815"),
"description" : "julia tried",
"name" : "Order1",
"brand_id" : ObjectId("5b0e52f058b8287a446f9f05")
}
There is also a collection for Brand and Category. This is the
Category collection
{
"_id" : ObjectId("5693d170a2191f9020b8c815"),
"name" : "Category1",
"created_at" : ISODate("2016-01-11T20:32:17.832+0000"),
"updated_at" : ISODate("2016-01-11T20:32:17.832+0000"),
}
Brand Collection
{
"_id" : ObjectId("5b0e52f058b8287a446f9f05"),
"name" : "brand1",
"description" : "brand1",
"updated_at" : ISODate("2017-07-05T09:18:13.951+0000"),
"created_at" : ISODate("2017-07-05T09:18:13.951+0000"),
}
Now after aggregation applied, it should result in below format:
{
'brands': [
{
_id: '*******'
name: 'brand1',
categories: [
{
_id: '*****',
name: 'category_name1',
orders: [
{
_id: '*****',
title: 'order1'
}
]
}
]
}
]
}
You can try below aggregation:
db.brand.aggregate([
{
$lookup: {
from: "order",
localField: "_id",
foreignField: "brand_id",
as: "orders"
}
},
{
$unwind: "$orders"
},
{
$lookup: {
from: "category",
localField: "orders.category_id",
foreignField: "_id",
as: "categories"
}
},
{
$unwind: "$categories"
},
{
$group: {
_id: "$_id",
name: { $first: "$name" },
description: { $first: "$description" },
updated_at: { $first: "$updated_at" },
created_at: { $first: "$created_at" },
categories: { $addToSet: "$categories" },
orders: { $addToSet: "$orders" }
}
},
{
$addFields: {
categories: {
$map: {
input: "$categories",
as: "category",
in: {
$mergeObjects: [
"$$category", {
orders: [ {
$filter: {
input: "$orders",
as: "order",
cond: { $eq: [ "$$category._id", "$$order.category_id" ] }
}
} ]
} ]
}
}
}
}
},
{
$project: {
orders: 0
}
}
])
Basically you have to use $lookup twice to "merge" data from all these collections based on brand_id and category_id fields. Since you expect orders in categories in brands you can use $unwind for both arrays and then use $group to get following shape:
{
"_id" : ObjectId("5b0e52f058b8287a446f9f05"),
"name" : "brand1",
"description" : "brand1",
"updated_at" : ISODate("2017-07-05T09:18:13.951Z"),
"created_at" : ISODate("2017-07-05T09:18:13.951Z"),
"categories" : [
{
"_id" : ObjectId("5693d170a2191f9020b8c814"),
"name" : "Category1",
"created_at" : ISODate("2016-01-11T20:32:17.832Z"),
"updated_at" : ISODate("2016-01-11T20:32:17.832Z")
}
],
"orders" : [
{
"_id" : ObjectId("5788fcd1d8159c2366dd5d93"),
"color" : "Blue",
"code" : "1",
"category_id" : ObjectId("5693d170a2191f9020b8c814"),
"description" : "julia tried",
"name" : "Order1",
"brand_id" : ObjectId("5b0e52f058b8287a446f9f05")
}
]
}
Now you have brand1 with all its subcategories and all orders that should be placed in one of those categories. The only thing is how to "nest" orders in categories. One way to do that might be $map where you can merge each category with all orders that match that category (using $mergeObjects you don't have to specify all properties from categories object).
To match category with orders you can perform $filter on orders array.
Then you can drop orders since those are nested into categories so you don't need them anymore.
EDIT: 3.4 version
In MongoDB 3.4 you can't use $mergeObjects so you should specify all properties for `categories:
db.brand.aggregate([
{
$lookup: {
from: "order",
localField: "_id",
foreignField: "brand_id",
as: "orders"
}
},
{
$unwind: "$orders"
},
{
$lookup: {
from: "category",
localField: "orders.category_id",
foreignField: "_id",
as: "categories"
}
},
{
$unwind: "$categories"
},
{
$group: {
_id: "$_id",
name: { $first: "$name" },
description: { $first: "$description" },
updated_at: { $first: "$updated_at" },
created_at: { $first: "$created_at" },
categories: { $addToSet: "$categories" },
orders: { $addToSet: "$orders" }
}
},
{
$addFields: {
categories: {
$map: {
input: "$categories",
as: "category",
in: {
_id: "$$category._id",
name: "$$category.name",
created_at: "$$category.created_at",
updated_at: "$$category.updated_at",
orders: [
{
$filter: {
input: "$orders",
as: "order",
cond: { $eq: [ "$$category._id", "$$order.category_id" ] }
}
}
]
}
}
}
}
},
{
$project: {
orders: 0
}
}
])
I am using MongoDB and HapiJs
I have 3 mongo db collections as follows:
companies: [
{_id: "autogeneratedId", name: "companyname1"},
{_id: "autogeneratedId", name: "companyname2"}
]
employees: [
{_id: "autogeneratedId", salary: 10000, employeename: "employeename1", "company_id": "_id of company"},
{_id: "autogeneratedId", salary: 50000, employeename: "employeename2", "company_id": "_id of company"},
{_id: "autogeneratedId", salary: 25000, employeename: "employeename3", "company_id": "_id of company"}
]
products: [
{_id: "autogeneratedId", price: 900, productname: "productname1", "company_id": "_id of company"},
{_id: "autogeneratedId", price: 400, productname: "productname2", "company_id": "_id of company"},
{_id: "autogeneratedId", price: 500, productname: "productname3", "company_id": "_id of company"}
]
I am trying to fetch the list of companies from companies collection. Now along with the list of companies I also want to fetch the list of products and employees of each corresponding company from collections products and employees based on _id of companies collection. Here is the code I am trying
db.getCollection('companies').aggregate([
{
$lookup:
{
from: "employees",
localField: "_id",
foreignField: "company_id",
as: "employees_array"
}
},
{
$lookup:
{
from: "products",
localField: "_id",
foreignField: "company_id",
as: "products_array"
}
},
])
But for some reason the above code returns an empty array in products_array and employees_array. So, can anyone tell me whats wrong am I doing over here?
Also in employees_array and products_array I want to fetch only data based on a condition like employees having salary >= 10000 and products having price >= 500. So how do I achieve this?
Here are the documents that I added:
companies
{
"_id" : ObjectId("5a3455b69beb3178555e98da"),
"name" : "companyname1"
}
{
"_id" : ObjectId("5a3455b69beb3178555e98dc"),
"name" : "companyname2"
}
employees
{
"_id" : ObjectId("5a34564c9beb3178555e990d"),
"salary" : 10000,
"employeename" : "employeename1",
"company_id" : ObjectId("5a3455b69beb3178555e98da")
}
{
"_id" : ObjectId("5a34564c9beb3178555e990f"),
"salary" : 50000,
"employeename" : "employeename2",
"company_id" : ObjectId("5a3455b69beb3178555e98da")
}
{
"_id" : ObjectId("5a34564c9beb3178555e9911"),
"salary" : 25000,
"employeename" : "employeename3",
"company_id" : ObjectId("5a3455b69beb3178555e98dc")
}
products
{
"_id" : ObjectId("5a3456929beb3178555e991e"),
"price" : 900,
"productname" : "productname1",
"company_id" : ObjectId("5a3455b69beb3178555e98da")
}
{
"_id" : ObjectId("5a3456929beb3178555e9920"),
"price" : 400,
"productname" : "productname2",
"company_id" : ObjectId("5a3455b69beb3178555e98dc")
}
{
"_id" : ObjectId("5a3456929beb3178555e9922"),
"price" : 500,
"productname" : "productname3",
"company_id" : ObjectId("5a3455b69beb3178555e98da")
}
When applying only your query:
db.getCollection('companies').aggregate([{
$lookup: {
from: "employees",
localField: "_id",
foreignField: "company_id",
as: "employees_array"
}
},
{
$lookup: {
from: "products",
localField: "_id",
foreignField: "company_id",
as: "products_array"
}
}
])
}
If your fields are named as you posted in your question then you should get results. Look for some type or something. I get the following results (which seem correct from what you describe):
{
"_id" : ObjectId("5a3455b69beb3178555e98da"),
"name" : "companyname1",
"employees_array" : [
{
"_id" : ObjectId("5a34564c9beb3178555e990d"),
"salary" : 10000,
"employeename" : "employeename1",
"company_id" : ObjectId("5a3455b69beb3178555e98da")
},
{
"_id" : ObjectId("5a34564c9beb3178555e990f"),
"salary" : 50000,
"employeename" : "employeename2",
"company_id" : ObjectId("5a3455b69beb3178555e98da")
}
],
"products_array" : [
{
"_id" : ObjectId("5a3456929beb3178555e991e"),
"price" : 900,
"productname" : "productname1",
"company_id" : ObjectId("5a3455b69beb3178555e98da")
},
{
"_id" : ObjectId("5a3456929beb3178555e9922"),
"price" : 500,
"productname" : "productname3",
"company_id" : ObjectId("5a3455b69beb3178555e98da")
}
]
}
{
"_id" : ObjectId("5a3455b69beb3178555e98dc"),
"name" : "companyname2",
"employees_array" : [
{
"_id" : ObjectId("5a34564c9beb3178555e9911"),
"salary" : 25000,
"employeename" : "employeename3",
"company_id" : ObjectId("5a3455b69beb3178555e98dc")
}
],
"products_array" : [
{
"_id" : ObjectId("5a3456929beb3178555e9920"),
"price" : 400,
"productname" : "productname2",
"company_id" : ObjectId("5a3455b69beb3178555e98dc")
}
]
}
If you want these salary >= 10000 and products having price >= 500 conditions you'd have to formulate your aggregation pipeline like this:
db.getCollection('companies').aggregate([{
$lookup: {
from: "employees",
localField: "_id",
foreignField: "company_id",
as: "employees_array"
}
},
{
$lookup: {
from: "products",
localField: "_id",
foreignField: "company_id",
as: "products_array"
}
},
{
$project: {
employees: {
$map: {
input: {
"$filter": {
"input": "$employees_array",
"as": "employee",
"cond": {
"$gte": ["$$employee.salary", 10000]
}
}
},
as: "emp",
in: {
"salary": "$$emp.salary",
"name": "$$emp.employeename"
// You can add more fields
}
}
},
products: {
$map: {
input: {
"$filter": {
"input": "$products_array",
"as": "product",
"cond": {
"$gte": ["$$product.price", 500]
}
}
},
as: "prd",
in: {
"price": "$$prd.price",
"productname": "$$prd.productname"
// you can add more fields
}
}
}
}
}
])
If some arrays don't satisfy this salary >= 10000 (for employees) or this price >= 500 (for products) it is possible that you get 0 results, but that is normal.