Join a collection to an existing aggregate query mongodb - node.js

I have the following collection structures in my mongodb database.
Orders
[
{
"_id":"order_abcd",
"name":"Order 1"
},
{
"_id":"order_defg",
"name":"Order 2"
}
]
Session
{
"_id":"session_abcd"
"orders": [ ObjectId("order_abcd"), ObjectId("order_defg") ]
}
Transactions
{
"_id":"transaction_abcd"
"id_session" : ObjectId("session_abcd")
}
What I am trying to achieve is a dataset that looks similar to this
[
{
"_id":"order_abcd",
"name":"Order 1",
"transaction":"transaction_abcd"
},
{
"_id":"order_defg",
"name":"Order 2",
"transaction":"transaction_abcd"
}
]
I do not have any input data other than a start date and an end date which will be used to filter orders, the query is mostly for reporting purposes so in effect I am trying to generate a query to fetch all orders between a given time period and attach the transaction id for each order to it.

We can use couple of $lookup (analogous to join with 2 tables in SQL) with $unwind at each stage to finally $project the key-value pair that is desired.
db.Session.aggregate([
{
$lookup: {
from: "Transactions",
localField: "_id",
foreignField: "id_session",
as: "transaction_info"
}
},
{ $unwind: "$transaction_info" },
{
$lookup: {
from: "Orders",
localField: "orders",
foreignField: "_id",
as: "order_info"
}
},
{ $unwind: "$order_info" },
{$project:{
_id:"$order_info._id",
name:"$order_info.name",
transaction:"$transaction_info._id"
}}
]).pretty();
Which gives an output:
{
"_id" : "order_abcd",
"name" : "Order 1",
"transaction" : "transaction_abcd"
},
{
"_id" : "order_defg",
"name" : "Order 2",
"transaction" : "transaction_abcd"
}
The unwind stages are used to explode the lookup and then cherry pick fields at final project stage.
++UPDATE++
Another option that probably can help reduce the 2nd stage lookup records since $match on dates of Orders can be applied to pass on filtered docs for next stage.
db.Session.aggregate([
{
$lookup: {
from: "Orders",
localField: "orders",
foreignField: "_id",
as: "order_info"
}
},
{ $unwind: "$order_info" },
{
$match: {} //filter on "order_info.property" (i:e; date,name,id)
},
{
$lookup: {
from: "Transactions",
localField: "_id",
foreignField: "id_session",
as: "transaction_info"
}
},
{ $unwind: "$transaction_info" },
{
$project: {
_id: "$order_info._id",
name: "$order_info.name",
transaction: "$transaction_info._id"
}
}
]).pretty();

Related

MongoDB Join on _id field from String to ObjectId aggregate $lookup

I have two collections.
Customers:
{
"_id" : ObjectId("584aac38686860d502929b8b"),
"name" : "user",
"email" : "user#gmail.com"
}
Posts:
{
"_id" : ObjectId("584aaca6686860d502929b8d"),
"title" : "Post",
"description":"description",
"user_id" : "584aac38686860d502929b8b"
}
I want to join this collection based on the user_id (from posts collection) - _id ( in customers collection).
I tried the below query:
dbo.collection('customers').aggregate([
{
$lookup:
{
from: 'posts',
localField: 'user_id',
foreignField: '_id',
as: 'posts'
}
}
])
but it's not working.
The output I am getting:
{
"_id": "584aac38686860d502929b8b",
"name": "user",
"email": "user#gmail.com",
"posts": []
}
From attached posts collection, user_id was a string but not ObjectId.
To compare, you have to convert user_id to ObjectId first.
db.customers.aggregate([
{
$lookup: {
from: "posts",
let: {
customerId: "$_id"
},
pipeline: [
{
$match: {
$expr: {
$eq: [
{
"$toObjectId": "$user_id"
},
"$$customerId"
]
}
}
}
],
as: "posts"
}
}
])
Sample Mongo Playground
Note: From your existing $lookup, you have reversed localField and foreignField.
Equality Match with a Single Join Condition
{
$lookup:
{
from: <collection to join>,
localField: <field from the input documents>,
foreignField: <field from the documents of the "from" collection>,
as: <output array field>
}
}

The Mongo query taking too much time to respond

I am working with node - js and mongoDB , all the queries are ok and working fine but at some point I am using a query used 4 lookups , also applied the matches based on those lookups , and applied the pagination+ sorting in the same query. But the main issue I am facing is the query taking time around 10-20 seconds to fetch the data from the database , which is really too long time period.
Here is the code snippet for the same
var products = await db.collection('catalog_products')
.aggregate([
{ $match: { categories: cat_id.toString(), status:1, verification_status:1}},
{ $lookup: { from: 'catalog_product_meta', localField: '_id', foreignField: 'product_id', as: 'meta' } }, { $unwind:"$meta" },
{ $lookup: { from: 'catalog_product_attributes', localField: '_id', foreignField: 'product_id', as: 'attributes' } }, { $unwind:"$attributes" },
{$match : {$and : [ { $or : [ { "attributes.attribute_value" : ObjectId("60f5626681cc91c83a34f6c8") }, { "attributes.attribute_value" : ObjectId("617285baaad0c6b9d269a6c5") } ] }, { $or : [ { "attributes.attribute_value" : ObjectId("61600701dc103aaf206165c3") } ] } ]}},
{ $lookup: { from: 'catalog_product_prices', localField: '_id', foreignField: 'product_id', as: 'prices' } }, { $unwind:"$prices" },
{ $match : {$and:{ 'prices.regular_price': { '$gte': 3109, '$lte': 15406 } }}},
{ $sort: sort},
{ $project: {
"_id" : 1,
"name" : "$meta.name",
"url_key" : 1,
"regular_price" : "$prices.regular_price",
"sale_price" : "$prices.sale_price",
} },
],{ "allowDiskUse" : true }).skip(36).limit(36).toArray();

get _id based on value from a collection for a array and insert in mongodb collection

Input:
[{"gradeName":1,"sectionName":"a","maximumStudents":12},{"gradeName":2,"sectionName":"b","maximumStudents":13}]
In mongodb how to get the _id of gradeName and sectionName from two collection and store in the third collection say mergeddetails
mergeddetails:
[{"grade_id":"60f819e04a9d43158cabad55","section_id":"60f819e04a9d43158cabad54","maximumStudents":12},{"grade_id":"60f819e04a9d43158cabad59","section_id":"60f819e04a9d43158cabad5b","maximumStudents":13}]
where grade_id is the _id of corresponding gradeName,section_id is the _id of corresponding sectionName
please help with with mongodb quire i am using nodejs to write the API
You need to use $lookup for both the collections
db.student.aggregate([
{ $lookup:
{ from: "grade",
localField: "gradeName",
foreignField: "grade",
as: "grade"
}
},
{$lookup:
{ from: "section",
localField: "sectionName",
foreignField: "section",
as: "section"
}
},
{ $unwind: "$grade" },
{ $unwind: "$section" },
{$project:
{
_id:0,
"grade_id":"$grade._id",
"section_id":"$section._id",
maxStudent:1
}
}
])

MongoDB $lookup in different collections by _id

I have 3 mongoDB collections
I need to aggregate them with $lookup operator but I didn't find anything/**or I'm bad looking **
1st one is suppliers
{
"_id" : ObjectId("111"), //for example, in db is mongodb ids
"name" : "supplier 1",
}
{
"_id" : ObjectId("222"),
"name" : "supplier 1",
}
2nd one is clients
{
"_id" : ObjectId("333"), //for example, in db is mongodb ids
"name" : "clients 1",
}
{
"_id" : ObjectId("444"),
"name" : "clients 2",
}
and 3rd is moves
{
"_id" : ObjectId("..."), //for example, in db is mongodb ids
"moveName" : "move 1",
"agent": ObjectId("111") // this is from suppliers collection
}
{
"_id" : ObjectId("..."),
"moveName" : "move 2",
"agent": ObjectId("333") // this one is from CLIENTS collection
}
so like output I need data like this
{
"_id" : ObjectId("..."), //for example, in db is mongodb ids
"moveName" : "move 1",
**"agent": supplier 1** // this is from suppliers collection
}
{
"_id" : ObjectId("..."),
"moveName" : "move 2",
**"agent": clients 1** // this one is from CLIENTS collection
}
back end is nodejs, I`m using mongoose, how I can search in 2nd collection if noresult in 1st?
const moves = await Move.aggregate([
{ $match: query }, // here all wokrs good
{
$lookup: {
from: 'clients',
localField: 'agent',
foreignField: '_id',
as: 'agent'
}
},{ $unwind: {path: "$agent" , preserveNullAndEmptyArrays: true} },
{
$lookup: {
from: 'suppliers',
localField: 'agent',
foreignField: '_id',
as: 'agent2'
}
},
{
$project: {
operationName: 1,
agent: {$ifNull: ['$agent.name', '$agent2.name']}
}
}
])
Thank You!
As suggested by #hhharsha36, we can use $facet operator which allows to run several pipelines within a single stage.
Explanation
facet
suppliers = $lookup suppliers collection and filter only matched results
clientes = $lookup clientes collection and filter only matched results
concatArrays = We concat suppliers and clients results into a single movies array
unwind = We flatten movies array [a, b, c] -> a
b
c
replaceWith = We replace the root element [movies:a, movies:b -> a, b]
mergeObject = allows us to pick the agent name (this way we avoid 1 more stage)
db.moves.aggregate([
{
$facet: {
suppliers: [
{
$lookup: {
from: "suppliers",
localField: "agent",
foreignField: "_id",
as: "agent"
}
},
{
$match: {
agent: {
$not: {
$size: 0
}
}
}
}
],
clients: [
{
$lookup: {
from: "clients",
localField: "agent",
foreignField: "_id",
as: "agent"
}
},
{
$match: {
agent: {
$not: {
$size: 0
}
}
}
}
]
}
},
{
$project: {
movies: {
"$concatArrays": [
"$clients",
"$suppliers"
]
}
}
},
{
$unwind: "$movies"
},
{
$replaceWith: {
"$mergeObjects": [
"$movies",
{
agent: {
"$arrayElemAt": [
"$movies.agent.name",
0
]
}
}
]
}
}
])
MongoPlayground
This aggregation query gives the desired result:
db.moves.aggregate([
{
$lookup: {
from: "suppliers",
localField: "agent",
foreignField: "_id",
as: "moves_sup"
}
},
{
$unwind: { path: "$moves_sup" , preserveNullAndEmptyArrays: true }
},
{
$lookup: {
from: "clients",
localField: "agent",
foreignField: "_id",
as: "moves_client"
}
},
{
$unwind: { path: "$moves_client" , preserveNullAndEmptyArrays: true }
},
{
$addFields: {
agent: {
$cond: [ { $eq: [ { $type: "$moves_sup" }, "object" ] },
"$moves_sup.name",
{ $cond: [ { $eq: [ { $type: "$moves_client" }, "object" ] }, "$moves_client.name", "undefined" ] }
] },
moves_client: "$$REMOVE",
moves_sup: "$$REMOVE"
}
},
])

mongodb (aggregation) - $lookup with the lookup result

i am new in aggregation framework and have the following problem/question.
My collections looks like this:
users
_id: ObjectId('604caef28aa89e769585f0c4')
baseData: {
firstName: "max"
username: "max_muster"
...
...
}
activitiesFeed
_id: ObjectId('604cb97c99bbd77b54047370')
userID: "604caef28aa89e769585f0c4" // the user id
activity: "604caf718aa89e769585f0c8" // the activity id
viewed: false
activities
_id: ObjectId('604caf718aa89e769585f0c8')
"baseData": {
"userID":"604caef28aa89e769585f0c4",
"type":0,
"title":"Alessandro send you a friend request.",
"description":"",
"actionID":"604caef28aa89e769585f0c4"
},
aggregate function
const response = ActivityFeed.aggregate([
{
$match: { userID: ObjectId(args.user) },
},
{
$lookup: {
from: 'activities',
localField: 'activity',
foreignField: '_id',
as: 'lookup_activities',
},
},
{ $unwind: '$activities' },
{
$lookup: {
from: 'users',
localField: 'lookup_activities.baseData.actionID',
foreignField: '_id',
as: 'lookup_users',
},
},
]);
How i can made a lookup with the first lookup result?
I made a lookup to the activities collection. This collection holds the id for the second lookup baseData.userID. But with this approach i have no result with the second lookup.
The reason why i made a join to the activities collection is that the actionID can hold a userID or a document id from another collection. It depend on the type in the activities collection.
Is it in the aggregation framework possible to made a lookup which depends on the type and of the previous lookup?
Thank you in advance.
All I did is fixing your aggregation. You unwind the wrong field and please make sure all your ID fields are ObjectId. I highly recommend using MongoDB Compass to assist your aggregation if you are new to the aggregation function. They have stage by stage assistance GUI which will really make your life easier.
mongoplayground
db.activitiesFeed.aggregate([
{
$match: {
userID: ObjectId("604caef28aa89e769585f0c4")
},
},
{
$lookup: {
from: "activities",
localField: "activity",
foreignField: "_id",
as: "lookup_activities",
},
},
{
$unwind: "$lookup_activities"
},
{
$lookup: {
from: "users",
localField: "lookup_activities.baseData.actionID",
foreignField: "_id",
as: "lookup_users",
},
},
])
Sample Used
db={
"users": [
{
_id: ObjectId("604caef28aa89e769585f0c4"),
baseData: {
firstName: "max",
username: "max_muster"
}
}
],
"activitiesFeed": [
{
_id: ObjectId("604cb97c99bbd77b54047370"),
userID: ObjectId("604caef28aa89e769585f0c4"),
activity: ObjectId("604caf718aa89e769585f0c8"),
viewed: false
}
],
"activities": [
{
_id: ObjectId("604caf718aa89e769585f0c8"),
"baseData": {
"userID": ObjectId("604caef28aa89e769585f0c4"),
"type": 0,
"title": "Alessandro send you a friend request.",
"description": "",
"actionID": ObjectId("604caef28aa89e769585f0c4")
},
}
]
}

Resources