MongoDB $lookup in different collections by _id - node.js

I have 3 mongoDB collections
I need to aggregate them with $lookup operator but I didn't find anything/**or I'm bad looking **
1st one is suppliers
{
"_id" : ObjectId("111"), //for example, in db is mongodb ids
"name" : "supplier 1",
}
{
"_id" : ObjectId("222"),
"name" : "supplier 1",
}
2nd one is clients
{
"_id" : ObjectId("333"), //for example, in db is mongodb ids
"name" : "clients 1",
}
{
"_id" : ObjectId("444"),
"name" : "clients 2",
}
and 3rd is moves
{
"_id" : ObjectId("..."), //for example, in db is mongodb ids
"moveName" : "move 1",
"agent": ObjectId("111") // this is from suppliers collection
}
{
"_id" : ObjectId("..."),
"moveName" : "move 2",
"agent": ObjectId("333") // this one is from CLIENTS collection
}
so like output I need data like this
{
"_id" : ObjectId("..."), //for example, in db is mongodb ids
"moveName" : "move 1",
**"agent": supplier 1** // this is from suppliers collection
}
{
"_id" : ObjectId("..."),
"moveName" : "move 2",
**"agent": clients 1** // this one is from CLIENTS collection
}
back end is nodejs, I`m using mongoose, how I can search in 2nd collection if noresult in 1st?
const moves = await Move.aggregate([
{ $match: query }, // here all wokrs good
{
$lookup: {
from: 'clients',
localField: 'agent',
foreignField: '_id',
as: 'agent'
}
},{ $unwind: {path: "$agent" , preserveNullAndEmptyArrays: true} },
{
$lookup: {
from: 'suppliers',
localField: 'agent',
foreignField: '_id',
as: 'agent2'
}
},
{
$project: {
operationName: 1,
agent: {$ifNull: ['$agent.name', '$agent2.name']}
}
}
])
Thank You!

As suggested by #hhharsha36, we can use $facet operator which allows to run several pipelines within a single stage.
Explanation
facet
suppliers = $lookup suppliers collection and filter only matched results
clientes = $lookup clientes collection and filter only matched results
concatArrays = We concat suppliers and clients results into a single movies array
unwind = We flatten movies array [a, b, c] -> a
b
c
replaceWith = We replace the root element [movies:a, movies:b -> a, b]
mergeObject = allows us to pick the agent name (this way we avoid 1 more stage)
db.moves.aggregate([
{
$facet: {
suppliers: [
{
$lookup: {
from: "suppliers",
localField: "agent",
foreignField: "_id",
as: "agent"
}
},
{
$match: {
agent: {
$not: {
$size: 0
}
}
}
}
],
clients: [
{
$lookup: {
from: "clients",
localField: "agent",
foreignField: "_id",
as: "agent"
}
},
{
$match: {
agent: {
$not: {
$size: 0
}
}
}
}
]
}
},
{
$project: {
movies: {
"$concatArrays": [
"$clients",
"$suppliers"
]
}
}
},
{
$unwind: "$movies"
},
{
$replaceWith: {
"$mergeObjects": [
"$movies",
{
agent: {
"$arrayElemAt": [
"$movies.agent.name",
0
]
}
}
]
}
}
])
MongoPlayground

This aggregation query gives the desired result:
db.moves.aggregate([
{
$lookup: {
from: "suppliers",
localField: "agent",
foreignField: "_id",
as: "moves_sup"
}
},
{
$unwind: { path: "$moves_sup" , preserveNullAndEmptyArrays: true }
},
{
$lookup: {
from: "clients",
localField: "agent",
foreignField: "_id",
as: "moves_client"
}
},
{
$unwind: { path: "$moves_client" , preserveNullAndEmptyArrays: true }
},
{
$addFields: {
agent: {
$cond: [ { $eq: [ { $type: "$moves_sup" }, "object" ] },
"$moves_sup.name",
{ $cond: [ { $eq: [ { $type: "$moves_client" }, "object" ] }, "$moves_client.name", "undefined" ] }
] },
moves_client: "$$REMOVE",
moves_sup: "$$REMOVE"
}
},
])

Related

The Mongo query taking too much time to respond

I am working with node - js and mongoDB , all the queries are ok and working fine but at some point I am using a query used 4 lookups , also applied the matches based on those lookups , and applied the pagination+ sorting in the same query. But the main issue I am facing is the query taking time around 10-20 seconds to fetch the data from the database , which is really too long time period.
Here is the code snippet for the same
var products = await db.collection('catalog_products')
.aggregate([
{ $match: { categories: cat_id.toString(), status:1, verification_status:1}},
{ $lookup: { from: 'catalog_product_meta', localField: '_id', foreignField: 'product_id', as: 'meta' } }, { $unwind:"$meta" },
{ $lookup: { from: 'catalog_product_attributes', localField: '_id', foreignField: 'product_id', as: 'attributes' } }, { $unwind:"$attributes" },
{$match : {$and : [ { $or : [ { "attributes.attribute_value" : ObjectId("60f5626681cc91c83a34f6c8") }, { "attributes.attribute_value" : ObjectId("617285baaad0c6b9d269a6c5") } ] }, { $or : [ { "attributes.attribute_value" : ObjectId("61600701dc103aaf206165c3") } ] } ]}},
{ $lookup: { from: 'catalog_product_prices', localField: '_id', foreignField: 'product_id', as: 'prices' } }, { $unwind:"$prices" },
{ $match : {$and:{ 'prices.regular_price': { '$gte': 3109, '$lte': 15406 } }}},
{ $sort: sort},
{ $project: {
"_id" : 1,
"name" : "$meta.name",
"url_key" : 1,
"regular_price" : "$prices.regular_price",
"sale_price" : "$prices.sale_price",
} },
],{ "allowDiskUse" : true }).skip(36).limit(36).toArray();

lookup with add extra field in mongodb

My OBJ
[{
_id:XXXXXXXXXX,
role:admin
},
{
_id:XXXXXXXXXX,
role:superUser
}]
and need results using aggregation how to solve this using aggregation
[{
name:'username'
role:'test'
}
]
I suppose you need the following
let db1 = db.get().collection(`temp1`);
let db2 = db.get().collection(`temp2`);
await db1.aggregate([
{
$lookup: {
from: "temp2",
localField: "_id", // field in the orders collection
foreignField: "_id", // field in the items collection
as: "users"
}
},
{
$replaceRoot: { newRoot: { $mergeObjects: [{ $arrayElemAt: ["$users", 0] }, "$$ROOT"] } }
},
{ $project: { users: 0 } }
]).toArray()

How to lookup inside lookup in MongoDB Aggregate?

I have a simple 3 collections. This bellow is their pseudocode. I want to get all shipments and for each shipment, I want to have all bids for that shipment and in each bid, I need userDetails object.
User: {
name: string,
}
Shipment: {
from: string,
to: string
}
Bid: {
amount: number,
shipmentId: Ref_to_Shipment
userId: Ref_to_User
}
This is what I have tried:
const shipments = await ShipmentModel.aggregate([
{
$lookup: {
from: "bids",
localField: "_id",
foreignField: "shipmentId",
as: "bids"
}
},
{
$lookup: {
from: "users",
localField: "bids.userId",
foreignField: "_id",
as: "bids.user"
}
}
])
And I got the following result:
[
{
"_id": "5fad2fc04458ac156531d1b1",
"from": "Belgrade",
"to": "London",
"__v": 0,
"bids": {
"user": [
{
"_id": "5fad2cdb4d19c80d1b6abcb7",
"name": "Amel",
"email": "Muminovic",
"password": "d2d2d2",
"__v": 0
}
]
}
}
]
I am trying to get all Shipments with their bids and users within bids. Data should look like:
[
{
"_id": "5fad2fc04458ac156531d1b1",
"from": "Belgrade",
"to": "London",
"__v": 0,
"bids": [
{
"_id": "5fad341887c2ae1feff73402",
"amount": 400,
"userId": "5fad2cdb4d19c80d1b6abcb7",
"shipmentId": "5fad2fc04458ac156531d1b1",
"user": {
"name": "Amel",
}
"__v": 0
}
]
}
]
Try with the following code:
const shipments = await ShipmentModel.aggregate([
{
$lookup: {
from: "bids",
localField: "_id",
foreignField: "shipmentId",
as: "bids"
}
},
{
$unwind: {
path: "$bids",
preserveNullAndEmptyArrays: true
}
},
{
$lookup: {
from: "users",
localField: "bids.userId",
foreignField: "_id",
as: "bids.user"
}
}
])
If you want to prevent null and empty arrays then set
preserveNullAndEmptyArrays: false
Try this query and chek if works and the behaviour is as you expected:
db.Shipment.aggregate([
{
$lookup: {
from: "Bid",
localField: "id",
foreignField: "shipmentId",
as: "bids"
}
},
{
$lookup: {
from: "user",
localField: "id",
foreignField: "id",
as: "newBids"
}
},
{
$project: {
"newBids.id": 0,
"newBids._id": 0,
}
},
{
$match: {
"bids.userId": 1
}
},
{
$addFields: {
"newBids": {
"$arrayElemAt": [
"$newBids",
0
]
}
}
},
{
$set: {
"bids.user": "$newBids"
}
},
{
$project: {
"newBids": 0
}
}
])
This query do your double $lookup and then a $project to delete the fields you don't want, and look for the userId to add the field user. As $lookup generate an array, is necessary use arrayElemAt to get the first position.
Then $set this value generated into the object as bids.user and remove the old value.
Note that I have used a new field id instead of _id to read easier the data.
Try this
I figured out it based on MongoDB $lookup on array of objects with reference objectId and in the answer from J.F. (data organization). Note that he used id instead of _id
The code is
db.Shipment.aggregate([
{
$lookup: {
from: "Bid",
localField: "id",
foreignField: "shipmentId",
as: "bids"
}
},
{
$lookup: {
from: "user",
localField: "bids.userId",
foreignField: "id",
as: "allUsers"
}
},
{
$set: {
"bids": {
$map: {
input: "$bids",
in: {
$mergeObjects: [
"$$this",
{
user: {
$arrayElemAt: [
"$allUsers",
{
$indexOfArray: [
"$allUsers.id",
"$$this.userId"
]
}
]
}
}
]
}
}
}
}
},
{
$unset: [
"allUsers"
]
},
// to get just one
//{
// $match: {
// "id": 1
// }
// },
])

Mongodb - Find count of distinct items after applying aggregate and match

Trying to figure out something from Mongo using mongoose in optimal way.
I have following documents
Regions
{
"_id" : ObjectId("5cf21263ff605c49cd6d8016"),
"name" : "Asia"
}
Countries can be part of multiple regions
{
"_id" : ObjectId("5d10a4ad80a93a1d7cd56cc6"),
"regions" : [
ObjectId("5d10a50080a93a1d7cd56cc7"),
ObjectId("5cf2126bff605c49cd6d8017")
],
"name" : "India"
}
Places belongs to one country
{
"_id" : ObjectId("5d11bb8180a93a1d7cd56d26"),
"name" : "Delhi",
"country" : ObjectId("5d136e7a4e480863a51c4056"),
}
Programs each in dayshows array represents one day. On a day show can cover multiple places.
{
"_id" : ObjectId("5d11cc9480a93a1d7cd56d31"),
"dayshows" : [
{
"_id" : ObjectId("5d11cc9480a93a1d7cd56d41"),
"places" : [
ObjectId("5d11bb8180a93a1d7cd56d26")
],
},
{
"_id" : ObjectId("5d11cc9480a93a1d7cd56d3c"),
"places" : [
ObjectId("5d11bb8180a93a1d7cd56d26"),
ObjectId("5d11bc7c80a93a1d7cd56d2e")
]
}
]
}
What am I trying to figure out?
For a given region, for each country in region which all places are covered and count of programs for each place. Using nodejs and mongoose.
Example
Input - Asia
Output
India
- Delhi (3)
- Mumbai (5)
Thailand
- Pattaya (2)
- Bangkok (5)
New to mongo.
You need to use $lookup to cross different collections.
Pipeline:
Stages 1-6 serves to get all related data.
(Optional) Stages 7-10 serves to transform aggregated data into key:pair object.
ASSUMPTION
Programs to visit 2 places counted as is (Place1: +1, Place2: +1)
You know how to execute MongoDB aggregation in node.js
db.Regions.aggregate([
{
$match: {
name: "Asia"
}
},
{
$lookup: {
from: "Countries",
let: {
region: "$_id"
},
pipeline: [
{
$match: {
$expr: {
$in: [
"$$region",
"$regions"
]
}
}
},
{
$lookup: {
from: "Places",
localField: "_id",
foreignField: "country",
as: "Places"
}
}
],
as: "Countries"
}
},
{
$unwind: "$Countries"
},
{
$unwind: "$Countries.Places"
},
{
$lookup: {
from: "Programs",
localField: "Countries.Places._id",
foreignField: "dayshows.places",
as: "Countries.Places.Programs"
}
},
{
$project: {
"name": 1,
"Countries.name": 1,
"Countries.Places.name": 1,
"Countries.Places.Programs": {
$size: "$Countries.Places.Programs"
}
}
},
{
$group: {
_id: {
name: "$name",
Countries: "$Countries.name"
},
Places: {
$push: {
k: "$Countries.Places.name",
v: "$Countries.Places.Programs"
}
}
}
},
{
$project: {
_id: 1,
Places: {
$arrayToObject: "$Places"
}
}
},
{
$group: {
_id: "$_id.name",
Countries: {
$push: {
k: "$_id.Countries",
v: "$Places"
}
}
}
},
{
$project: {
_id: 0,
name: "$_id",
Countries: {
$arrayToObject: "$Countries"
}
}
}
])
MongoPlayground

Aggregate document multilevel

Now consider the case , i have one document containing below collection like structure.
Below is the order collection
{
"_id" : ObjectId("5788fcd1d8159c2366dd5d93"),
"color" : "Blue",
"code" : "1",
"category_id" : ObjectId("5693d170a2191f9020b8c815"),
"description" : "julia tried",
"name" : "Order1",
"brand_id" : ObjectId("5b0e52f058b8287a446f9f05")
}
There is also a collection for Brand and Category. This is the
Category collection
{
"_id" : ObjectId("5693d170a2191f9020b8c815"),
"name" : "Category1",
"created_at" : ISODate("2016-01-11T20:32:17.832+0000"),
"updated_at" : ISODate("2016-01-11T20:32:17.832+0000"),
}
Brand Collection
{
"_id" : ObjectId("5b0e52f058b8287a446f9f05"),
"name" : "brand1",
"description" : "brand1",
"updated_at" : ISODate("2017-07-05T09:18:13.951+0000"),
"created_at" : ISODate("2017-07-05T09:18:13.951+0000"),
}
Now after aggregation applied, it should result in below format:
{
'brands': [
{
_id: '*******'
name: 'brand1',
categories: [
{
_id: '*****',
name: 'category_name1',
orders: [
{
_id: '*****',
title: 'order1'
}
]
}
]
}
]
}
You can try below aggregation:
db.brand.aggregate([
{
$lookup: {
from: "order",
localField: "_id",
foreignField: "brand_id",
as: "orders"
}
},
{
$unwind: "$orders"
},
{
$lookup: {
from: "category",
localField: "orders.category_id",
foreignField: "_id",
as: "categories"
}
},
{
$unwind: "$categories"
},
{
$group: {
_id: "$_id",
name: { $first: "$name" },
description: { $first: "$description" },
updated_at: { $first: "$updated_at" },
created_at: { $first: "$created_at" },
categories: { $addToSet: "$categories" },
orders: { $addToSet: "$orders" }
}
},
{
$addFields: {
categories: {
$map: {
input: "$categories",
as: "category",
in: {
$mergeObjects: [
"$$category", {
orders: [ {
$filter: {
input: "$orders",
as: "order",
cond: { $eq: [ "$$category._id", "$$order.category_id" ] }
}
} ]
} ]
}
}
}
}
},
{
$project: {
orders: 0
}
}
])
Basically you have to use $lookup twice to "merge" data from all these collections based on brand_id and category_id fields. Since you expect orders in categories in brands you can use $unwind for both arrays and then use $group to get following shape:
{
"_id" : ObjectId("5b0e52f058b8287a446f9f05"),
"name" : "brand1",
"description" : "brand1",
"updated_at" : ISODate("2017-07-05T09:18:13.951Z"),
"created_at" : ISODate("2017-07-05T09:18:13.951Z"),
"categories" : [
{
"_id" : ObjectId("5693d170a2191f9020b8c814"),
"name" : "Category1",
"created_at" : ISODate("2016-01-11T20:32:17.832Z"),
"updated_at" : ISODate("2016-01-11T20:32:17.832Z")
}
],
"orders" : [
{
"_id" : ObjectId("5788fcd1d8159c2366dd5d93"),
"color" : "Blue",
"code" : "1",
"category_id" : ObjectId("5693d170a2191f9020b8c814"),
"description" : "julia tried",
"name" : "Order1",
"brand_id" : ObjectId("5b0e52f058b8287a446f9f05")
}
]
}
Now you have brand1 with all its subcategories and all orders that should be placed in one of those categories. The only thing is how to "nest" orders in categories. One way to do that might be $map where you can merge each category with all orders that match that category (using $mergeObjects you don't have to specify all properties from categories object).
To match category with orders you can perform $filter on orders array.
Then you can drop orders since those are nested into categories so you don't need them anymore.
EDIT: 3.4 version
In MongoDB 3.4 you can't use $mergeObjects so you should specify all properties for `categories:
db.brand.aggregate([
{
$lookup: {
from: "order",
localField: "_id",
foreignField: "brand_id",
as: "orders"
}
},
{
$unwind: "$orders"
},
{
$lookup: {
from: "category",
localField: "orders.category_id",
foreignField: "_id",
as: "categories"
}
},
{
$unwind: "$categories"
},
{
$group: {
_id: "$_id",
name: { $first: "$name" },
description: { $first: "$description" },
updated_at: { $first: "$updated_at" },
created_at: { $first: "$created_at" },
categories: { $addToSet: "$categories" },
orders: { $addToSet: "$orders" }
}
},
{
$addFields: {
categories: {
$map: {
input: "$categories",
as: "category",
in: {
_id: "$$category._id",
name: "$$category.name",
created_at: "$$category.created_at",
updated_at: "$$category.updated_at",
orders: [
{
$filter: {
input: "$orders",
as: "order",
cond: { $eq: [ "$$category._id", "$$order.category_id" ] }
}
}
]
}
}
}
}
},
{
$project: {
orders: 0
}
}
])

Resources