I am trying to generate data for a heat map on my dashboard. I need to group the data by days i.e Monday, Tuesday e.t.c and then group the data by time intervals. I am able to group the data by days. But the time interval part doesnt work. I am using MongoDB.
Sample of my Schema is shown below
{
"createdAt": "2020-06-30T22:47:32+00:00",
"day": "Wednesday",
"user": {
"$oid": "5ec51d59ddfb380017649591"
},
"country": "NG",
"city": "",
"userType": "superadmin",
"fullName": "Jane Doe",
"__v": 0
}
This is what I could come up with so far
let heatMap = await await Session.aggregate([
{
$group: {
_id: "$day",
count: { $sum: 1 },
},
},
]);
This groups by day for me already.
I need to further group by intervals of hours
Sample Expected Data
[
{
_id: "Sunday",
intervals: [
{
interval1: "12am-6am",
count: 34,
},
{
interval2: "6am-12noon",
count: 44,
},
],
},
{
_id: "Monday",
intervals: [
{
interval1: "12am-6am",
count: 34,
},
{
interval2: "6am-12noon",
count: 44,
},
],
},
];
You can use this pipeline that utilizes $mod with some date operators to achieve the result you want.
The strategy is first to group by the intervals and the finally by the day of the week as this simplifies the process.
db.collection.aggregate([
{
"$group": {
"_id": {
"$toDate": {
"$subtract": [
{
"$toLong": {
$toDate: "$createdAt"
}
},
{
"$mod": [
{
"$toLong": {
$toDate: "$createdAt"
}
},
21600000
]
}
]
}
},
count: {
$sum: 1
}
}
},
{
$group: {
_id: {
day: {
$dayOfWeek: "$_id"
},
hour: {
$hour: "$_id"
}
},
count: {
$sum: "$count"
},
interval: {
$first: "$_id"
}
}
},
{
$group: {
_id: "$_id.day",
intervals: {
$push: {
count: "$count",
interval: "$interval"
}
}
}
}
])
Notice that the result is not exactly how you want, i.e _id: 3 is equal to _id: "Tuesday" As Mongo does not provide any out of the box conversions you'll have to do it yourself either in code or by using something like $switch
Mongo Playground
Related
Similar to another question I had (Here). But now I'm trying to count unique and total events on daily basis for each event type, based on the following data shape:
{
username: "jack",
events: [
{
eventType: "party",
createdAt: "2022-01-23T10:26:11.214Z",
visitorInfo: {
visitorId: "87654321-0ebb-4238-8bf7-87654321"
}
},
{
eventType: "party",
createdAt: "2022-01-23T10:26:11.214Z",
visitorInfo: {
visitorId: "87654321-0ebb-4238-8bf7-87654321"
}
},
{
eventType: "party",
createdAt: "2022-01-23T10:26:11.214Z",
visitorInfo: {
visitorId: "01234567-0ebb-4238-8bf7-01234567"
}
},
{
eventType: "meeting",
createdAt: "2022-01-23T12:26:11.214Z",
visitorInfo: {
visitorId: "87654321-0ebb-4238-8bf7-87654321"
}
},
{
eventType: "meeting",
createdAt: "2022-01-23T11:26:11.214Z",
visitorInfo: {
visitorId: "87654321-0ebb-4238-8bf7-87654321"
}
},
{
eventType: "meeting",
createdAt: "2022-01-23T12:26:11.214Z",
visitorInfo: {
visitorId: "01234567-0ebb-4238-8bf7-01234567"
}
},
{
eventType: "party",
createdAt: "2022-01-30T10:26:11.214Z",
visitorInfo: {
visitorId: "12345678-0ebb-4238-8bf7-12345678"
}
},
{
eventType: "party",
createdAt: "2022-01-30T10:16:11.214Z",
visitorInfo: {
visitorId: "12345678-0ebb-4238-8bf7-12345678"
}
},
{
eventType: "meeting",
createdAt: "2022-01-30T12:36:11.224Z",
visitorInfo: {
visitorId: "12345678-0ebb-4238-8bf7-12345678"
}
},
{
eventType: "meeting",
createdAt: "2022-01-30T11:46:11.314Z",
visitorInfo: {
visitorId: "12345678-0ebb-4238-8bf7-12345678"
}
}
]
}
I'm trying to count events (all and unique ones based on visitorId) on date (daily).
This is what I have so far (thanks to #R2D2's guide on the approach):
Event.aggregate([
{ $match: { username: 'jack' } },
{ $unwind: "$events" },
{
$project: {
totalPartyEvents: {
$cond: [
{
$eq: ["$events.eventType", "party"],
},
1,
0,
],
},
uniquePartyEvents: { // where I'm stuck. I need to count unique events based on visitorId on current date for 'party' event type.
$cond: [
{
$eq: ["$events.eventType", "party"],
},
1,
0,
],
},
totalMeetingEvents: {
$cond: [
{
$eq: ["$events.eventType", "meeting"],
},
1,
0,
],
},
uniqueMeetingEvents: { // do the same for other events. maybe there's a better way to combine these (with facets).
$cond: [
{
$eq: ["$events.eventType", "meeting"],
},
1,
0,
],
},
date: "$events.createdAt",
},
},
{
$group: {
_id: {
$dateToString: { format: "%Y-%m-%d", date: "$date" },
},
totalPartyEvents: {
$sum: "$totalMeetingEvents",
},
uniquePartyEvents: {
$sum: "$totalMeetingEvents",
},
totalMeetingEvents: {
$sum: "$totalMeetingEvents",
},
uniqueMeetingEvents: {
$sum: "$uniqueMeetingEvents",
},
},
},
{
$project: {
date: "$_id",
uniquePartyEvents: 1,
totalPartyEvents: 1,
totalMeetingEvents:1,
uniqueMeetingEvents: 1,
},
},
{
$group: {
_id: "0",
dateAndEventFrequency: {
$push: "$$ROOT",
},
},
},
{
$project: {
_id: 0,
dateAndEventFrequency: 1,
},
},
]);
I tried using $addToSet but it's not used with $project (it works with $group).
Any new approach is welcome based on the data shape and the desired result I'm expecting. I used $project because I was already using it.
Basically what I'm hoping to get in the end:
dateAndEventFrequency: [
{
_id: "2022-01-23",
totalPartyEvents: 3,
uniquePartyEvents: 2,
totalMeetingEvents: 3,
uniqueMeetingEvents: 2,
date: "2022-01-23",
},
{
_id: "2022-01-30",
totalPartyEvents: 2,
uniquePartyEvents: 1,
totalMeetingEvents: 2,
uniqueMeetingEvents: 1,
date: "2022-01-30",
},
]
I'm using Mongoose and Nodejs. Any help or guidance is appreciated. Thanks!
mongo playground
db.collection.aggregate([
{
$match: {
username: "jack"
}
},
{
"$unwind": "$events"
},
{
"$match": {
"events.eventType": {
"$in": [
"meeting",
"party"
]
}
}
},
{
"$group": {
"_id": {
date: {
"$dateToString": {
format: "%Y-%m-%d",
date: "$events.createdAt"
}
},
"visitorId": "$events.visitorInfo.visitorId",
"eventType": "$events.eventType"
},
"count": {
"$sum": 1
}
}
},
{
"$group": {
"_id": {
"date": "$_id.date",
"eventType": "$_id.eventType"
},
"uniqueTotal": {
"$sum": 1
},
total: {
"$sum": "$count"
}
}
},
{
"$group": {
"_id": "$_id.date",
"partyUniqueTotal": {
"$sum": {
"$cond": [
{
$eq: [
"$_id.eventType",
"party"
],
},
"$uniqueTotal",
0
]
}
},
"totalPartyEvents": {
"$sum": {
"$cond": [
{
$eq: [
"$_id.eventType",
"party"
],
},
"$total",
0
]
}
},
"meetingUniqueTotal": {
"$sum": {
"$cond": [
{
$eq: [
"$_id.eventType",
"meeting"
],
},
"$uniqueTotal",
0
]
}
},
"totalmeetingEvents": {
"$sum": {
"$cond": [
{
$eq: [
"$_id.eventType",
"meeting"
],
},
"$total",
0
]
}
}
}
}
])
I have the following MongoDB schema:
const userSchema = new mongoose.Schema({
email: {
type: String,
required: [true, 'Email is required.']
},
transactions: [
{
categoryName: {
type: String,
required: [true, 'Category name in transaction is required.']
},
categoryType: {
type: String,
required: [true, 'Category type in transaction is required.']
},
amount: {
type: Number,
required: [true, 'Transaction amount is required.']
}
}
]})
transactions.categoryType can only be Income or Expense. Now per queried _id, I want to return the ratio/percentage of transactions.CategoryName per Income and Expense. Meaning if I have the following data:
{
"_id": 000001,
"email": "asdasd#email.com"
"transactions": [
{
"categoryName": "Food",
"categoryType": "Expense",
"amount": 200
},
{
"categoryName": "Rent",
"categoryType": "Expense",
"amount": 1000
},
{
"categoryName": "Salary",
"categoryType": "Income",
"amount": 15000
}
]
}
the result that I would want is:
{ "email": "asdasd#email.com",
"Income": [["Salary", 100]],
"Expense": [["Food", 16.67],["Rent",83.33]],
}
Now, I have the following query:
return User.aggregate([
{ $match: { _id : ObjectId(request.params.id) } },
{ $unwind : "$transactions"},
{ $group : { _id : { type: "$transactions.categoryType" },
total: {$sum : "$transactions.amount"},
transactionsArray: { $push: "$transactions"}
}
},
{ $project: {
_id: 0,
transactionsArray:1,
type: "$_id.type",
total:1
}
}
])
which returns a data like this:
[
{
"total": 1200,
"transactions": [
{
"categoryName": "Food",
"categoryType": "Expense",
"amount": 200,
},
{
"categoryName": "Rent",
"categoryType": "Expense",
"amount": 1000,
}
],
"type": "Expense"
},
{
"total": 15000,
"transactions": [
{
"categoryName": "Salary",
"categoryType": "Income",
"amount": 15000,
}
],
"type": "Income"
}
]
Now, I do not know how am I going to further process the result set to divide the transactions.amount by the total to get the result that I want.
You may go with multiple steps in aggregations
$unwind to deconstruct the array
$group- first group to group by _id and $categoryType. So we can get the total amount and an amount for particular transaction. This helps to calculate the ratio.
$map helps to loop over the array and calculate the ratio
$reduce- You need comma separated string array of objects. So loop it and get the structure.
$group to group by _id only so we can get the key value pair of category type and Income/Expense when we push
$replaceRoot to make the $grp object as root which should be merged with already existing fields ($mergeObjects)
$project for remove unwanted fields
Here is the code
db.collection.aggregate([
{ "$unwind": "$transactions" },
{
"$group": {
"_id": { id: "$_id", catType: "$transactions.categoryType" },
"email": { "$first": "$email" },
"amount": { "$sum": "$transactions.amount" },
"category": {
$push: { k: "$transactions.categoryName", v: "$transactions.amount" }
}
}
},
{
$addFields: {
category: {
$map: {
input: "$category",
in: {
k: "$$this.k",
v: {
"$multiply": [
{ "$divide": [ "$$this.v","$amount" ]},
100
]
}
}
}
}
}
},
{
"$addFields": {
category: {
"$reduce": {
"input": "$category",
"initialValue": [],
"in": {
"$concatArrays": [
[
[ "$$this.k", { $toString: "$$this.v" } ]
],
"$$value"
]
}
}
}
}
},
{
"$group": {
"_id": "$_id.id",
"email": { "$first": "$email" },
"grp": { "$push": { k: "$_id.catType", v: "$category" } }
}
},
{
"$replaceRoot": {
"newRoot": {
"$mergeObjects": [ { "$arrayToObject": "$grp" }, "$$ROOT" ]
}
}
},
{ "$project": { grp: 0 } }
])
Working Mongo playground
I have the following structure in my collection (you don't have to mind the status) :
{
"_id": {
"$oid": "5e6355e71b14ee00175698cb"
},
"finance": {
"expenditure": [
{
"status": true,
"_id": { "$oid": "5e63562d1b14ee00175698df" },
"amount": { "$numberInt": "100" },
"category": "Sport"
},
{
"status": true,
"_id": { "$oid": "5e6356491b14ee00175698e0" },
"amount": { "$numberInt": "200" },
"category": "Sport"
},
{
"status": true,
"_id": { "$oid": "5e63565b1b14ee00175698e1" },
"amount": { "$numberInt": "50" },
"category": "Outdoor"
},
{
"status": true,
"_id": { "$oid": "5e63566d1b14ee00175698e2" },
"amount": { "$numberInt": "400" },
"category": "Outdoor"
}
]
}
}
My previos command was this:
User.aggregate([
{ $match: {_id: req.user._id} },
{ $unwind: '$finance.expenditure' },
{ $match: {'finance.expenditure.status': true} },
{ $sort: {'finance.expenditure.currentdate': -1} },
{
$group: {
_id: '$_id',
expenditure: { $push: '$finance.expenditure' }
}
}
])
With this I just get every single expenditure back.
But now I want to group the expenditures by their category and sum up the amount of every single expenditure for their group.
So it should look like this:
{ "amount": 300 }, "category": "Sport" },
{ "amount": 450 }, "category": "Outdoor" }
Thanks for your help
Instead of grouping on _id field group on category field & sum amount field:
db.collection.aggregate([
{ $match: {_id: req.user._id}},
{
$unwind: "$finance.expenditure"
},
{
$match: {
"finance.expenditure.status": true
}
},
{
$sort: {
"finance.expenditure.currentdate": -1
}
},
{
$group: {
_id: "$finance.expenditure.category",
amount: {
$sum: "$finance.expenditure.amount"
}
}
},
{
$project: {
_id: 0,
category: "$_id",
amount: 1
}
}
])
Test : MongoDB-Playground
I Have a bunch of sensordata stored in mongoDB. They are stored like this:
{
"data": [
{
"date": ISODate("2020-02-08T18:06:25.507+00:00"),
"temperature": 20.3,
"humidity": 53.7
},
{
"date": ISODate("2020-02-08T18:07:25.507+00:00"),
"temperature": 21,
"humidity": 54
}
]
}
The day-field is generated by new Date() with JavaScript.
Now i just want the get all the data and convert the "date"-field to a time-field. The result should look like this:
{
"data": [
{
"date": "18:06:25",
"temperature": 20.3,
"humidity": 53.7
},
{
"date": "18:07:25",
"temperature": 21,
"humidity": 54
}
]
}
So is there a way to convert every "date"-field in the array to a "time"-field by using db.collection.aggregate?
I tried using this:
db.collection.aggregate([
{},
{
"$project": {
"data.date": { $dateToString: { format: "%H:%M:%S",date: "$date" } },
"daydata.temperature": 1,
"daydata.humidity": 1
}
}
])
I know it doesnt work, because i dont have any "date"-field outside of "data". But i dont know how to reach the date-field of every data-object and convert it.
You can use $unwind then $project then $group in aggregate.
db.data.aggregate([
{ $unwind: "$data" },
{
$project: {
"data.date": {
$dateToString: { format: "%H:%M:%S", date: "$data.date" },
},
"data.temperature": 1,
"data.humidity": 1,
},
},
{
$group: {
_id: "$_id",
data: { $push: "$data" },
},
},
]);
Also, date should be ISODate
{
"data": [
{
"date": ISODate("2020-02-08T18:06:25.507+00:00"),
"temperature": 20.3,
"humidity": 53.7
},
{
"date": ISODate("2020-02-08T18:07:25.507+00:00"),
"temperature": 21,
"humidity": 54
}
]
}
Since the data.date field is a string, use the sub-string operator to extract the time part of the date field.
db.test.aggregate( [
{
$unwind: "$data"
},
{
$addFields: {
"data.time": { $substrCP: [ "$data.date", 11, 8 ] }
}
},
{
$project: { "data.date": 0 }
},
{
$group: {
_id: "$_id",
data: { $push: "$data" },
// other_fld: { $first: "$other_fld" }
}
}
] ).pretty()
here is the query
[
{
"$project": {
"formattedDate": {
"$dateToString": { "format": "%Y-%m-%d", "date": "$ceatedAt" }
},
"createdAtMonth": { "$month": "$ceatedAt" },
"rating": 1
}
},
{
"$group": {
"_id": "$formattedDate",
"average": { "$avg": "$rating" },
"month": { "$first": "$createdAtMonth" },
}
}
]
I need the date in timestamp. How to do that?
Mongodb 4.0 has introduced $toLong aggregation which convert date to timestamp
db.collection.aggregate([
{ "$project": {
"createdAt": {
"$toLong": "$createdAt"
}
}}
])
You can try it here
Use $subtract arithmetic aggregation operator with your Date as minuend and new Date("1970-01-01") as subtrahend.
db.collection.aggregate(
{
$project: { "timestamp": { $subtract: [ "$createdAt", new Date("1970-01-01") ] } }
}
);
For document
{ "_id": 1, "createdAt": ISODate("2016-09-01T14:35:14.952Z") }
the result is
{ "_id": 1, "timestamp": NumberLong("1472740514952") }
If you want to group both by timestamp and (year, month, date) you can divide timestamp by the amount of milliseconds in a day, so that it will be unique for each day (and not for each millisecond)
db.collection.aggregate(
{
$project:
{
"timestampByDay":
{
$floor:
{
$divide:
[
{ $subtract: [ "$createdAt", new Date("1970-01-01") ] },
24 * 60 * 60 * 1000
]
}
},
"date": "$createdAt"
}
},
{
$group:
{
"_id": "$timestampByDay",
"date": { $first: "$date" }
}
}
);
If you want to update timestamp. with the current date and time use the below query.
db.getCollection('movie').update(
{"applicationId":"2b5958d9629026491c30b42f2d5256fa8","type":"shortcut"},
{$set : {userName:"vipin+testkm23052020#applozic.com",created_at: NumberLong(new Date()),"updated_at":NumberLong(new Date()),"docIndex":UUID()}}, {multi:true, upsert: false}
)