I am trying to get some filtered data and it's total count. I want to do both job within single query, so how can I do this. Below is my code.
var SubId = 1;
var TypeId = 1;
var lookup = {
$lookup:
{
from: 'sub_types',
localField: 'sub_id',
foreignField: 'sub_id',
as: 'sub_category'
}
};
var unwind = { $unwind: "$sub_category" };
var project = {
"ques_id": 1,
"ques_txt": 1,
"ans_txt": 1,
"ielts_sub_id": 1,
"ielts_tags_id": 1,
};
var match = {
"sub_category.type_id": parseInt(TypeId),
"sub_category.sub_id": parseInt(SubId),
"status": 1
};
ieltsmongoose.collection('ques').aggregate([
lookup, unwind,
{
$match: match
},
{
$project: project
}
]).limit(max_row).toArray(async function (error, Ques) {....});
Now I want to get count with this same query like
{
$count: "totalcount"
},
You can use another aggregate stage called group like this to achieve your goal:
{ "$group": {
"_id": null,
"count": { "$sum": 1},
"data": { "$push": "$$ROOT" }
}}
your aggregation would be like :
ieltsmongoose.collection('ques').aggregate([
lookup, unwind,
{
$match: match
},
{
$project: project
},
{ "$group": {
"_id": null,
"count": { "$sum": 1},
"data": { "$push": "$$ROOT" }
}
}
])
also you should use limit and offset as a aggregation stage with $limit and $skip
Related
So what I want to do is group all documents having same hash whose count is more than 1 and only keep the oldest record according to startDate
My db structure is as follows:
[{
"_id": "82bacef1915f4a75e6a18406",
"Hash": "cdb3d507734383260b1d26bd3edcdfac",
"duration": 12,
"price": 999,
"purchaseType": "Complementary",
"startDate": {
"$date": {
"$numberLong": "1656409841000"
}
},
"endDate": {
"$date": {
"$numberLong": "1687859441000"
}
}
}]
I was using this query which I created
db.Mydb.aggregate([
{
"$group": {
_id: {hash: "$Hash"},
dups: { $addToSet: "$_id" } ,
count: { $sum : 1 }
}
},{"$sort":{startDate:-1}},
{
"$match": {
count: { "$gt": 1 }
}
}
]).forEach(function(doc) {
doc.dups.shift();
db.Mydb.deleteMany({
_id: {$in: doc.dups}
});
})
this gives a result like this:
{ _id: { hash: '1c01ef475d072f207c4485d0a6448334' },
dups:
[ '6307501ca03c94389f09b782',
'6307501ca03c94389f09b783',
'62bacef1915f4a75e6a18l06' ],
count: 3 }
The problem with this is that the _id's in dups array are random everytime I run this query i.e. not sorted according to startDate field.
What can be done here?
Any help is appreciated. Thanks!
After $group stage, startDate field will not pre present in the results, so you can not sort based on that field. So, as stated in the comments, you should put $sort stage first in the Aggregation pipeline.
db.Mydb.aggregate([
{
"$sort": { startDate: -1}
},
{
"$group": {
_id: {hash: "$Hash"},
dups: { $addToSet: "$_id" } ,
count: { $sum : 1 }
},
{
"$match": { count: { "$gt": 1 }
}
]
Got the solution. I was using $addToSet in the group pipeline stage which does not allow duplicate values. Instead, I used $push which allows duplicate elements in the array or set.
I am a newbie to MongoDB. I have a use case where the mongo query should be converted to aggregate query.
I have the following two collections:
items: {_id: "something", "name": "raj", "branch": "IT", subItems: ["subItemId","subItemId"]}
subItems: {_id: "something", "city": "hyd", list: ["adsf","adsf"]}
Query passed by the user is:
{
"query": { "name": "raj", "subItems.loc": "hyd" },
"sort": { "name": 1 },
"projection": { "_id": 0, "branch": 1, "subItems.loc": 1 }
}
I am able to create a dynamic query in a following way:
let itemConditions: any[] = [];
let subItemConditions: any[] = [];
let itemProjection: {[k:string]: any} = {};
let subItemProjection: {[k:string]: any} = {};
subItemConditions.push({$in: ["$_id", "$$subItems"]}); // Default condition to search files in the subItems collection
let isAnysubItemProj = false;
Object.keys(reqQuery).forEach(prop => {
let value = reqQuery[prop];
if(prop.includes('subItems.')) {
key = prop.split(".")[1];
if(key === '_id') value = new ObjectId(value);
subItemConditions.push({$eq: [`$${prop}`, value]});
return;
}
itemConditions.push({$eq: [`$${prop}`, value]});
});
if(config.projection)
Object.keys(config.projection).forEach(prop => {
if(prop.includes('subItems.')) {
isAnysubItemProj = true;
const key = prop.split(".")[1];
subItemProjection[key] = config.projection[prop];
return;
}
itemProjection[prop] = config.projection[prop];
});
if(isAnysubItemProj) itemProjection['subItems'] = 1;
let subItemPipeline: any[] = [];
subItemPipeline.push(
{ $match: {
$expr: {
$and: subItemConditions
}
}
});
if(Object.keys(subItemProjection).length)
subItemPipeline.push({$project: subItemProjection});
let query: any[] = [
{
$match: {
$expr : {
$and: itemConditions
}
}
},
{
$addFields: {
subItems: {
$map: {
input: "$subItems",
as: "id",
in: { $toObjectId: "$$id" }
}
}
}
},
{
$lookup: {
from: "subItems",
let: {subItems: "$subItems"},
pipeline: subItemPipeline,
as: "subItems"
}
}
];
if(config.sort && Object.keys(config.sort).length) query.push({$sort: config.sort});
if(Object.keys(itemProjection).length) query.push({$project: itemProjection});
const items = await collection.aggregate(query).toArray();
The above code will work only for the comparison of equality for items and subItems separately, but the user can send different types of queries like:
{
"query": { $or: [{"name": "raj"}, {subItems: {$gt: { $size: 3}}}], "subItems.city": "hyd" },
"sort": { "name": 1 },
"projection": { "_id": 0, "branch": 1, "subItems.loc": 1 }
}
{
"query": { $or: [{"name": "raj"}, {"subItems.city": {"$in" : ["hyd", "ncr"]}}], "subItems.list": {"$size": 2} },
"sort": { "name": 1 },
"projection": { "_id": 0, "branch": 1, "subItems.loc": 1 }
}
Is there any easy way to convert this normal MongoDB query into an aggregate query or is there any other approach to implement this...??
I am trying to modify the above dynamic query to work for any queries passed by the user but it is becoming difficult to handle all queries.
Is there any better approach to handle this situation like changing the query passed by the user or the way of handling it on the server-side or how I should change my code to support all types of queries passed by the user..??
Any help will be appreciated
If this is really your input
input = {
"query": { "name": "raj", "subItems.loc": "hyd" },
"sort": { "name": 1 },
"projection": { "_id": 0, "branch": 1, "subItems.loc": 1 }
}
Then the aggregation pipeline would be simply:
let pipeline = [
{$match: input.query},
{$sort: input.sort},
{$project: input.projection}
]
db.collection.aggregate(pipeline)
I am trying to using aggregate function for fetching data from two documents. I am able to do it but i am finding a solution how can i apply $project in lookup table only
below is my approach
app.get('/getAllDetailById',(req,res)=>{
if(db){
// lookup
db.collection("points").aggregate(
[
{ "$addFields": { "enquiry_by": { "$toObjectId": "$enquiry_by" }}},
{
"$lookup" : {
from: "user",
localField: "enquiry_by",
foreignField: "_id",
as: "userDetails"
}
},
{ $unwind: "$userDetails"},
]
).toArray()
.then(result=>{
console.log(result[0])
}).catch(err=>{
res.send(err)
})
}
})
What i want is get all fields from points table and from user table i just want name and username. I have used $project but than its return only fields defined in this.
{ $project: {"userDetails.name":1, "userDetails.username":1,"_id":0} }
Is there any way that $project can be applied separately for user table
You can use pipeline in the lookup if you are using mongodb >= 3.6: https://docs.mongodb.com/manual/reference/operator/aggregation/lookup/#join-conditions-and-uncorrelated-sub-queries
So your code will look like:
app.get('/getAllDetailById',(req,res)=>{
if(db){
// lookup
db.collection("points").aggregate(
[
{ "$addFields": { "enquiry_by": { "$toObjectId": "$enquiry_by" }}},
{
"$lookup" : {
from: "user",
let: { "enquiry_by": "$enquiry_by" },
pipeline: [
{
"$match": {
"$expr": {
"$eq": ["$_id", "$$enquiry_by"]
}
},
"$project": {
"$name": 1,
"$username": 1,
}
},
],
as: "userDetails"
}
},
{ $unwind: "$userDetails"},
]
).toArray()
.then(result=>{
console.log(result[0])
}).catch(err=>{
res.send(err)
})
}
})
I have a visits collection where I successfully count the number of visits per location
Visits model:
{
"_id": {
"$oid": "5a3969e2f4ea3e33ac5a523d"
},
"locationId": "5a395ccf210a1d35d0df4a58"
}
locationId above is of type 'Object' as I learned lookup localField and foreignField must be of same type
nodejs code =>
let sort = { "count": -1, "locationId": 1 };
Visit.aggregate([
{
$match:{
$and: [
{ accountId: req.session.passport.user },
{
'details.dateTimeIn': {
$gte: new Date(dateFrom), $lte: new Date(dateTo)
}
}
]
}
},
{
"$group": {
//_id: name,
_id: "$locationId",
count: { $sum: 1 }
}
},
{ $sort: sort }
])
Output is half ok:
[
{
"_id":"5a395ccf210a1d35d0df4a58",
"count":20
}
]
Instead of showing location id id like to show location name. Schema for locations collection is:
{
"_id": {
"$oid": "5a395ccf210a1d35d0df4a58"
"name": "Tower A",
"__v": 0
}
}
Research suggests I need to use $lookup to get that JOIN effect
So I tried
{
"$lookup": {
from: "locations",
localField: "_id",
foreignField: "_id",
as: "locationdetails"
}
}
but the match seems broken. The closest I got was a list of all locations in 'locationdetails'
But with code above here is the empty locationdetails
[
{
"_id":"5a395ddf1d221918d0041313",
"count":20,
"locationdetails":[
]
}
]
What am I missing ?
My Collection JSON
[
{
"_id" : 0,
"finalAmount":40,
"payment":[
{
"_id":0,
"cash":20
},
{
"_id":1,
"card":20
}
]
},
{
"_id" : 1,
"finalAmount":80,
"payment":[
{
"_id":0,
"cash":60
},
{
"_id":1,
"card":20
}
]
},
{
"_id" : 2,
"finalAmount":80,
"payment":[
{
"_id":0,
"cash":80
}
]
}
]
I want to have the amount, cash and card group wise using aggregation framework. Can anyone help?
Please consider my _id as an ObjectId for demo purpose as I have given 0 and 1. I am using Node Js and MongoDB and I want the expected output in just one query as follows:
Expected Output:
{
"cash":160,
"card":40,
"total":200,
"count":3
}
You could try running the following aggregation pipeline, although there might be some performance penalty or potential aggregation pipeline limits with huge datasets since your initial pipeline tries to group all the documents in the collection to get the total document count and the amount as well as pushing all the documents to a temporary list, which may affect performance down the pipeline.
Nonetheless, the following solution will yield the given desired output from the given sample:
collection.aggregate([
{
"$group": {
"_id": null,
"count": { "$sum": 1 },
"doc": { "$push": "$$ROOT" },
"total": { "$sum": "$finalAmount" }
}
},
{ "$unwind": "$doc" },
{ "$unwind": "$doc.payment" },
{
"$group": {
"_id": null,
"count": { "$first": "$count" },
"total": { "$first": "$total" },
"cash": { "$sum": "$doc.payment.cash" },
"card": { "$sum": "$doc.payment.card" }
}
}
], function(err, result) {
console.log(result);
});
When running on big datasets, this problem might be more suitable, more fast to solve with a map reduce operation, since the result is one singel aggregated result.
var map = function map(){
var cash = 0;
var card = 0;
for (i in this.payment){
if(this.payment[i].hasOwnProperty('cash')){
cash += this.payment[i]['cash']
}
if(this.payment[i].hasOwnProperty('card')){
card += this.payment[i]['card']
}
}
var doc = {
'cash': cash,
'card': card,
};
emit(null, doc);
};
var reduce = function(key, values){
var total_cash = 0;
var total_card = 0;
var total = 0;
for (i in values){
total_cash += values[i]['cash']
total_card += values[i]['card']
}
var result = {
'cash': total_cash,
'card': total_card,
'total': total_cash+ total_card,
'count': values.length
};
return result
};
db.runCommand({"mapReduce":"test", map:map, reduce:reduce, out:{replace:"test2"}})
result:
db.test2.find().pretty()
{
"_id" : null,
"value" : {
"cash" : 160,
"card" : 40,
"total" : 200,
"count" : 3
}
}