Data not getting fetched using aggregation query in nodejs - node.js

I tried below code but only first match is checked and displayed,others displayed as object, why I am unable to see it in console. I have 3 collections student subject teacher, also made schema for the same. tried aggregation
Student.aggregate([
{
$match: { name: 'abcd'}
},
{
$lookup:
{
from:'teachers',
pipeline: [{ $match: { name: 'pqrs' } },],
as: "teacherLookup"
}
},
{
$lookup:
{
from:'subjects',
pipeline: [{ $match: { name: 'computer' } }],
as: "subjectLookup"
}
}
]).then(function (res) {
console.log(res);
res.forEach(function(students){
let id = students._id;
console.log(id+ ' got id ')
}
output
student
name:'abcd' -- its fetched and other two not displaying values only shows object
teacherLookup: [ [Object] ]
subjectLookup: [ [Object] ]

You were there just to project something you have to use $project stage.
Here I'm adding the query:
Student.aggregate([
{
$match: { name: 'abcd'}
},
{
$lookup:{
from:'teachers',
pipeline: [
{
$match: { name: 'pqrs' }
},
{
$project:{
"_id":1
}
}
],
as: "teacherLookup"
}
},
{
$lookup:
{
from:'subjects',
pipeline: [
{
$match: { name: 'computer' }
},
{
$project:{
"_id":1
}
}
],
as: "subjectLookup"
}
}
])
For more about $project refer here.
Hope this will help :)

Related

Mongodb $lookup inside $addfield?

I have a collection named users, and this is how one specific user will look like:
{
_id: 'Object ID',
name: 'String',
cart: [
{
product_id: 'Product object ID',
quantity: 'Number',
},
...
],
}
I want my desired results to look like this:
{
_id: 'Object ID',
name: 'String',
cart: [
{
product_id: 'Product object ID',
quantity: 'Number',
product_details: {
'all the details of the product from Products collection which matches the product_id',
},
},
...
],
}
I tried adding addFields into lookup but it's getting too complicated and doesn't work as desired. What's the best way to aggregate this?
You can achieve this in several different ways, here's what I consider to be the most simple:
db.users.aggregate([
{
"$lookup": {
"from": "products",
let: {
cart: "$cart"
},
pipeline: [
{
$match: {
$expr: {
$in: [
"$_id",
"$$cart.product_id"
]
}
}
},
{
$replaceRoot: {
newRoot: {
"$mergeObjects": [
"$$ROOT",
{
"$arrayElemAt": [
{
$filter: {
input: "$$cart",
cond: {
$eq: [
"$_id",
"$$this.product_id"
]
}
}
},
0
]
}
]
}
}
}
],
"as": "cart"
}
}
])
Mongo Playground

MongoDB (Mongoose) - Get only records where nested item is equal to a value

I searched a lot, tried several ways, but nothing works.
I have this in mongo:
{
id: ObjectId("1234567890"),
answers: [{
id: 111,
deleted:0
},
{
id: 222,
deleted:0
},
{
id: 333,
deleted:1
}]
},
{
id: ObjectId("0987654321"),
answers: [{
id: 111,
deleted:0
},
{
id: 222,
deleted:1
},
{
id: 333,
deleted:1
}]
}
I want the document with ObjectId("1234567890"), and only the answers with delete = 1 ( only the id 333).
I have tryied this:
var query = chatbotMongoModel.find(
{ _id: "1234567890" },
{ answers: {$elemMatch: {deleted: "1"}}}
)
but returns all the answers.. Could you give me some help plase?
thanks!
Rafael
One way of doing this is using mongodb aggregation framework.
var result = await chatbotMongoModel.aggregate([
{
$match: {
id: "1234567890"
}
},
{
"$unwind": {
path: "$answers"
}
},
{
$match: {
"answers.deleted": 1
}
},
{
$group: {
_id: "$id",
answers: {
$push: "$answers"
},
allFields: {
$first: "$$ROOT"
}
}
},
{
"$replaceRoot": {
"newRoot": {
"$mergeObjects": [
"$allFields",
{
"answers": "$answers"
}
]
}
}
}
])
Note that in your sample documents you have id but you use _id in your query, they must match. Also deleted data type is number in sample documents, but you use string value in your query.
Running mongoplayground.
Another alternative solution:
var result = await chatbotMongoModel.aggregate([
{
$match: {
id: "1234567890"
}
},
{
$addFields: {
"answers": {
$filter: {
input: "$answers",
as: "answer",
cond: {
$eq: [
"$$answer.deleted",
1
]
}
}
}
}
}
])
mongoplayground
Thanks for the answers! I found a solution, and im posting here if someone needs this too...
var query = model.aggregate([
{ $match: { '_id': mongoose.Types.ObjectId("1234567890") } },
{
$project: {
answer: {
$filter: {
input: "$answer",
as: "f",
cond: { $eq: ["$$f.deleted", 1] }
}
}
}
}
])
regards!

Group $lookup results within inner nested array, and keep outer nested array and its contents intact in MongoDB

I have a collection which comprises of three level array nesting as shown below
_id: ObjectID('abc'),
sections: [
{
sectionId: "sec0",
sectionName: "ABC",
contents: [
{
contentId: 0,
tasks: [
{
taskId: ObjectID('task1')
}
//May contain 1-100 tasks
],
contentDescription: "Content is etc",
}
]
}
]
Sections is an array of objects which contains an object each with sectionId, and contents array which is an array of objects comprising of contentId, contentDescription, and nested array of tasks which comprises of an object containing a taskId.
I am applying $lookup operator in order to join nested tasks array with tasks collection but I am facing a problem in document duplication as shown below.
_id: ObjectID('abc'),
sections: [
{
sectionId: "sec0",
sectionName: "ABC",
contents: [
{
contentId: 0,
tasks: [
{
//Task Document of ID 1
}
],
contentDescription: "Content is etc",
}
]
}
]
_id: ObjectID('abc'),
sections: [
{
sectionId: "sec0",
sectionName: "ABC",
contents: [
{
contentId: 0,
tasks: [
{
//Task Document of ID 2
}
],
contentDescription: "Content is etc",
}
]
}
]
Whereas the desired output is as follows
_id: ObjectID('abc'),
sections: [
{
sectionId: "sec0",
sectionName: "ABC",
contents: [
{
contentId: 0,
tasks: [
{
//Task Document of ID 1
},
{
//Task Document of ID 2
},
{
//Task Document of ID 3
}
],
contentDescription: "Content is etc",
}
]
}
]
In the collection, a sections array might contain multiple section object which might contain multiple contents and so on and so forth.
The schema in question is temporary as our company is currently migrating from an existing database to MongoDB, so architectural refactoring is not possible atm and I need to work with existing schema design from different database.
I tried the following way
const contents= await sections.aggregate([
{
$match: { _id: id},
},
{ $unwind: '$sections' },
{
$unwind: {
path: '$sections.contents',
preserveNullAndEmptyArrays: true,
},
},
{
$unwind: {
path: '$sections.contents.tasks',
preserveNullAndEmptyArrays: true,
},
},
{
$lookup: {
from: 'tasks',
let: { task_id: '$sections.contents.tasks.taskId' },
pipeline: [
{ $match: { $expr: { $eq: ['$_id', '$$task_id'] } } },
],
as: 'sections.contents.tasks',
},
},
{
$addFields: {
'sections.contents.tasks': {
$arrayElemAt: ['$sections.contents.tasks', 0],
},
},
},
{
$group: {
_id: '$_id',
exam: { $push: '$sections.contents.tasks' },
},
},
]);
And I am also unable to use $group aggregation operator like
$group: {
_id: '$_id',
sections: {
sectionId : { $first: '$sectionId' },
sectionName: { $first: '$sectionName' },
contents: {
contentId: { $first: '$contentId' },
task: { $push: $sections.contents.tasks }
}
},
},
Any help or directions will be appreciated, I also searched on SO, and found this but couldn't understand the following part
{"$group":{
"_id":{"_id":"$_id","mission_id":"$missions._id"},
"agent":{"$first":"$agent"},
"title":{"$first":"$missions.title"},
"clients":{"$push":"$missions.clients"}
}},
{"$group":{
"_id":"$_id._id",
"missions":{
"$push":{
"_id":"$_id.mission_id",
"title":"$title",
"clients":"$clients"
}
}
}}
So you're very close to the final solution, a good "rule" that's good to remember is if you unwind x times you need to group x to restore the original structure properly, like so:
db.collection.aggregate([
{
$match: {
_id: id
},
},
{
$unwind: "$sections"
},
{
$unwind: {
path: "$sections.contents",
preserveNullAndEmptyArrays: true,
},
},
{
$unwind: {
path: "$sections.contents.tasks",
preserveNullAndEmptyArrays: true,
},
},
{
$lookup: {
from: "tasks",
let: {
task_id: "$sections.contents.tasks.taskId"
},
pipeline: [
{
$match: {
$expr: {
$eq: [
"$_id",
"$$task_id"
]
}
}
},
],
as: "sections.contents.tasks",
},
},
{
$addFields: {
"sections.contents.tasks": {
$arrayElemAt: [
"$sections.contents.tasks",
0
],
},
},
},
{
$group: {
_id: {
contentId: "$sections.contents.contentId",
sectionId: "$sections.sectionId",
sectionName: "$sections.sectionName",
originalId: "$_id"
},
tasks: {
$push: "$sections.contents.tasks"
},
contentDescription: {
$first: "$sections.contents.contentDescription"
},
}
},
{
$group: {
_id: {
sectionId: "$_id.sectionId",
sectionName: "$_id.sectionName",
originalId: "$_id.originalId"
},
contents: {
$push: {
contentId: "$_id.contentId",
tasks: "$tasks",
contentDescription: "$contentDescription"
}
}
}
},
{
$group: {
_id: "$_id.originalId",
sections: {
$push: {
sectionId: "$_id.sectionId",
sectionName: "$_id.sectionName",
contents: "$contents"
}
}
}
}
])
Mongo Playground
However your pipeline could be made a little cleaner as it has 1 redundant $unwind stage that also adds a redundant $group stage. I won't post the entire fixed pipeline here as it's already a long post but feel free to check it out here: Mongo Playground fixed

I want to display only one product image

This is Code in node js
const result = await OrderDB.aggregate([
{ $match: { _id: mongoose.Types.ObjectId(id) } },
{
$lookup: {
from: 'products',
localField: 'product',
foreignField: '_id',
as: 'productDetail',
},
},
{
$project: {
productDetail: {
name: 1,
price: 1,
productImage: 1,
},
},
},
])
This is the response of code
{
"message": "Get Order Successfully",
"result": [
{
"_id": "5ff47348db5f5917f81871aa",
"productDetail": [
{
"name": "Camera",
"productImage": [
{
"_id": "5fe9b8a26720f728b814e246",
"img": "uploads\\product\\7Rq1v-app-7.jpg"
},
{
"_id": "5fe9b8a26720f728b814e247",
"img": "uploads\\product\\FRuVb-app-8.jpg"
}
],
"price": 550
}
]
}
]
}
I want to display only one productImage from the response using nodejs and mongoose
This is using in aggregate projection
I was use $arrayElemAt but it is don't work
I also use $first but it is don't work
so projection method I use to display only one *productImage*
Data looks like multi level nested.
You have array of results, each result contains array of productDetails
play
You need to unwind the data to get the first productImage
db.collection.aggregate([
{
"$unwind": "$result"
},
{
"$unwind": "$result.productDetail"
},
{
$project: {
pImage: {
"$first": "$result.productDetail.productImage"
}
}
}
])
With the above response
db.collection.aggregate([
{
"$project": {
productDetails: {
$map: {
input: "$productDetail",
in: {
"$mergeObjects": [
"$$this",
{
productImage: {
"$arrayElemAt": [
"$$this.productImage",
0
]
}
}
]
}
}
}
}
}
])
Working Mongo playground

MongoDB search and pagination Aggregation Performance issue

I'm new in node js and MongoDB. I'm working on MongoDB search and pagination which is working good, but I have an issue with performance. it is taking too much time in counting and search records.
if I use small word to search then it works faster, if I use "long string" or "no record in database" then it takes too much time which is 50 to 186.30 seconds. (it is too much time, I'm expecting it to be 1 to 2 seconds).
I have more than 15,00,000 data on my record.
If I do not include count of the search word. it is takes 0.20 to 1.5 seconds, but when I count records while searching word it takes 25.0 to 35.0 seconds.
I have no idea how to decrease this time for counting records with the search word(query optimization).
I tried max level of query optimization.
I have also tried with
{
$count: "passing_scores"
}
but no change on time. I'm stuck on it. I have to decrease the time of count with the search word.
SQL Query for example
SELECT * FROM `post`
Left JOIN catagory ON post.catid=catagory.id
WHERE post_name LIKE '%a%' OR post_data LIKE '%a%' OR tags LIKE '%a%' OR post_url LIKE '%a%'
NODE and MongoDB
PostObj.count({},function(err,totalCount) {
if(err) {
response = {"error" : true,"message" : "Error fetching data"}
}
PostObj.aggregate([
{ $lookup:
{
from: 'catagories',
localField: 'catagory.catagory_id',
foreignField: '_id',
as: 'catagories_data'
}
},
{
$match:
{
$or: [
{"catagories_data.catagory_name": { $regex: new RegExp(search_data)}},
{"postname": { $regex: new RegExp(search_data) }},
{"posturl": { $regex: new RegExp(search_data) }},
{"postdata": { $regex: new RegExp(search_data) }},
{"tags": { $regex: new RegExp(search_data) }}
]
}
},
{ $limit : search_limit },
{ $skip : search_skip },
{ $group : { _id : "$_id", postname: { $push: "$postname" } , posturl: { $push: "$posturl" } } }
]).exec(function (err, data){
//end insert log data
if(err) {
response = {"error" : true,"message" :err};
}
if(search_data != "")
{
// count record using search word
PostObj.aggregate([
{ $lookup:
{
from: 'catagories',
localField: 'catagory.catagory_id',
foreignField: '_id',
as: 'catagories_data'
}
},
{
$match:
{
$or: [
{"catagories_data.catagory_name": { $regex: new RegExp(search_data)}},
{"postname": { $regex: new RegExp(search_data) }},
{"posturl": { $regex: new RegExp(search_data) }},
{"postdata": { $regex: new RegExp(search_data) }},
{"tags": { $regex: new RegExp(search_data) }}
]
}
},
{ $group: { _id: null, myCount: { $sum: 1 } } },
{ $project: { _id: 0 } }
]).exec(function (err, Countdata){
res.json({
sEcho : req.body.draw,
iTotalRecords: Countdata.myCount,
iTotalDispla,yRecords: Countdata.myCount,
aaData: data
});
}
res.json({
sEcho : req.body.draw,
iTotalRecords: totalPages,
iTotalDisplayRecords: totalPages,
aaData: data
});
});
});
Also, I have to try this way but it is tack 35.0 to 49.0 seconds more than 1st code.
PostObj.aggregate([
{ $lookup:
{
from: 'catagories',
localField: 'catagory.catagory_id',
foreignField: '_id',
as: 'catagories_data'
}
},
{
$match:
{
$or: [
{"catagories_data.catagory_name": { $regex: new RegExp(search_data)}},
{"postname": { $regex: new RegExp(search_data) }},
{"posturl": { $regex: new RegExp(search_data) }},
{"postdata": { $regex: new RegExp(search_data) }},
{"tags": { $regex: new RegExp(search_data) }}
]
}
},
{ '$facet' : {
metadata: [ { $count: "total" }, { $addFields: { page: NumberInt(3) } } ],
data: [ { $skip: 20 }, { $limit: 10 } ] // add projection here wish you re-shape the docs
} }
] )
If I do not use search word it is work good. I have an issue with when searching any word(count of records of that work without skip and limit)
collection data
Post
{
"_id": ObjectId("5d29bd7609f28633f38ccc13"),
"postname": "this is some data ",
"tags " : "
Damita,
Caro,
Leontyne,
Theodosia,
Vyky ",
"postdata ": "Berry Samara Kellia Rebekah Linette Hyacinthie Joelly Micky Tomasina Christian Fae Doralynn Chelsea Aurie Gwendolyn Tate
Cairistiona Ardys Aubrie Damita Olga Kelli Leone Marthena Kelcy
Cherlyn Molli Pris Ginelle Sula Johannah Hedwig Adelle Editha Lindsey
Loleta Lenette Ann Heidie Drona Charlena Emilia Manya Ketti Dorthea
Jeni Lorene Eolanda Karoly Loretta Marylou Tommie Leontyne Winny Cyb
Violet Pavia Karen Idelle Betty Doloritas Judye Aretha Quinta Billie
Vallie Fiona Letty Gates Shandra Rosemary Dorice Doro Coral Tove Crin
Bobbe Kristan Tierney Gianina Val Daniela Kellyann Marybeth Konstance
Nixie Andeee Jolene Patrizia Carla Arabella Berna Roseline Lira Cristy
Hedi Clem Nerissa ",
"catagory " : [
{ "catagory_id " : [ ObjectId("5d29bd7509f28633f38ccbfd")]},
{ "catagory_id": [ ObjectId("5d29bd7509f28633f38ccbfd") ]}],
"createby": "5d22f712fe481b2a9afda4aa"
}
catagory
{
"_id": ObjectId("5d29bc271a68fb333531f6a1"),
"catagory_name": "Katharine",
"catagory_description": "Katharine"
}
Any solution for it?
If in your case, your regex is just looking for a (or few) word(s), then it would be better to use $text instead of $regex. $text can use text index and is thus much faster. In terms of MySQL, $text is LIKE and $regex is REGEXP. Since in your example mysql query you are using LIKE, I'm pretty confident you can go for $text instead of $regex, in your mongo query as well.
You need to have (if not already) a compound "text" index on your fields - (postname, tags, postdata and posturl).
db.POST.createIndex(
{
postname: "text",
tags: "text",
posturl: "text",
postdata: "text"
}
)
There are some tips that i can suggest you try.
1: POST collection
it seems you are storing only category_id inside your category array of objects property, which you should avoid.
instead what you should do is as below.
create new property post_id inside category collection instead of array of object of category in post collection in [ high performance approach ].
OR
convert category property of post collection form array of object to simple array. [ average performance ].
Ex: category: [ ObjectId("5d29bd7509f28633f38ccbfd", ObjectId("5d29bd7509f28633f38ccbfd", ObjectId("5d29bd7509f28633f38ccbfd"];
definitely in both the cases post_id or category property must be indexed.
2: lookup
instead using simple lookup pipeline you should use pipeline approach
Eg:
NOT GOOD.
$lookup:{
from: 'catagories',
localField: 'catagory.catagory_id', // BAD IDEA //
foreignField: '_id',
as: 'catagories_data'
},
GOOD.
$lookup:{
from: 'catagories',
localField: '_id',
foreignField: 'post_id', // GOOD IDEA
as: 'catagories_data'
},
EVEN BETTER
$lookup:{
let : { post_id: "$_id" },
from: 'catagories',
pipeline:[
{
$match: {
$expr: {
$and: [
{ $eq: ["$post_id", "$$post_id"], },
]
}
},
},
{
$match: {
$or: [
// AVOID `new` keyword if you can do such;
// and create indexes for the same;
{ "catagory_name": { $regex: `^${search_data}` } },
{ "postname": { $regex: `^${search_data}` } },
{ "posturl": { $regex: `^${search_data}` } },
{ "postdata": { $regex: `^${search_data}` } },
{ "tags": { $regex: `^${search_data}` } }
]
}
}
],
as: 'catagories_data'
},
After All facet pipeline seems fine to me.
'$facet' : {
metadata: [ { $count: "total" }, { $addFields: { page: NumberInt(3) } } ],
data: [ { $skip: 20 }, { $limit: 10 } ] // add projection here wish you re-shape the docs
}
Other aspects of slowdown query depends on
configuration of your backend server and database server.
distance between frontend -> backend -> database server.
incoming and outgoing request per second.
internet connection of course
Complete Query will look like this
PostObj.aggregate([
{
$lookup: {
let: { post_id: "$_id" },
from: 'categories',
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ["$post_id", "$$post_id"], },
]
}
},
},
{
$match: {
$or: [
// AVOID `new` keyword if you can do such;
// and create indexes for the same;
{ "catagory_name": { $regex: `^${search_data}` } },
{ "postname": { $regex: `^${search_data}` } },
{ "posturl": { $regex: `^${search_data}` } },
{ "postdata": { $regex: `^${search_data}` } },
{ "tags": { $regex: `^${search_data}` } }
]
}
}
],
as: "catagories_data"
}
},
{
'$facet': {
metadata: [{ $count: "total" }, { $addFields: { page: NumberInt(3) } }],
catagories_data: [{ $skip: 0 }, { $limit: 10 }]
}
}
])

Resources