How to generate time series data from existing data in mongodb - node.js

Question

I have to generate this report.
In my MongoDB database I have a collection of orders like this:
[
{
_id: "mongoId", // 5f5ea6276ba53b06944de28c
createdAt: "2020-09-15T23:07:19.370Z",
totalPrice: 34, // is calculated from the client (quantity * price)
orderDetail: [
{
_id: "product-A-Id", // 5f5ea403e91ed91a44b62c92
quantity: 4,
price: 5.5,
},
{
_id: "product-B-Id",
quantity: 1,
price: 3.5,
},
{
_id: "product-C-Id",
quantity: 1,
price: 8.5,
},
],
},
{
_id: "mongoId",
createdAt: "2020-09-15T23:08:20.370Z",
totalPrice: 15.5,
orderDetail: [
{
_id: "product-C-Id",
quantity: 3,
price: 3,
},
{
_id: "product-D-Id",
quantity: 1,
price: 6.5,
},
],
},
{
_id: "mongoId",
createdAt: "2020-09-15T23:09:25.370Z",
totalPrice: 22.5,
orderDetail: [
{
_id: "product-D-Id",
quantity: 5,
price: 4.5,
},
],
},
]
To make this I have to generate time series data each two hours from timestamp now (in every request), the example of response desired is this:
[
{
id: "sales",
data: [
{
x: "00:00",
y: 150,
},
{
x: "22:00",
y: 100,
},
{
x: "20:00",
y: 150,
},
{
x: "18:00",
y: 50,
},
{
x: "16:00",
y: 100,
},
],
},
]
Using nodejs and express like framework I could generate sales of the last 2 hours:
const valueDateRange = 2 * 60 * 60 * 1000; // 2 hours
const currentPeriod = new Date(new Date().getTime() - valueDateRange);
// The last 2 hours sales
const calculateTotalSales = await Order.aggregate([
{
$match: { createdAt: { $gte: currentPeriod } },
},
{
$group: { _id: null, TotalAmount: { $sum: "$totalPrice" } },
},
]);
But now how to generate the time series data each 2 hours, so much thanks for the attention

Answer 1

The cleanest way I've found is to rework each docs date using the modulo operator to group them by hour blocks. You can easily change if you need bigger blocks in the future.
https://mongoplayground.net/p/aYAJKL_5dMD (I added extra sample data)
db.orders.aggregate([
{$addFields: {
date: {
$let: {
vars: {
hour: {$hour: '$createdAt'},
remainder: {$mod: [
{$hour: '$createdAt'},
2 // Two hour blocks, can be 2,3,4,6,8,12
]},
},
in: {
$dateFromParts: {
year: {$year: '$createdAt'},
month: {$month: '$createdAt'},
day: {$dayOfMonth: '$createdAt'},
hour: {$subtract: ['$$hour', '$$remainder']}
}
}
}
}
}},
{$group: {
_id: '$date',
x: {$last: '$date'},
y: {$sum: '$totalPrice'}
}}
]);
Update:
After reading your question again, I think your looking for total per hour regardless of the day. You can do so like this:
https://mongoplayground.net/p/cpW9JKllDIN
const totals = await db.orders.aggregate([
{$addFields: {
hour: {
$let: {
vars: {
hour: {$hour: '$createdAt'},
remainder: {$mod: [
{$hour: '$createdAt'},
2 // Two hour blocks, can be 2,3,4,6,8,12
]},
},
in: {$subtract: ['$$hour', '$$remainder']}
}
}
}},
{$group: {
_id: '$hour',
x: {$last: '$hour'},
y: {$sum: '$totalPrice'}
}}
])
Then to include hours that have no sales you can map an array:
let points = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22].map(x => {
let total = totals.find(t => t.x === x);
return {
x: `${x < 10 ? `0${x}` : x}:00`,
y: total ? total.y : 0
};
});

Answer 2

After trying many times, I ended up with this code, I hope it helps someone in the future, to prove it, make sure you have the updated data at the time you make the request, otherwise it would result 0 because compare the dates with the current moment.
Mongo Playgournd
node.js: v12.5.0
express: "^4.17.1"
Mongodb version v4.2.3
router.get("/orders", async (req, res) => {
let valueDateRange = 24 * 60 * 60 * 1000; // 24 hours
const current = new Date();
const previous = new Date(new Date().getTime() - valueDateRange);
try {
const order = await Order.aggregate([
{
$match: {
createdAt: { $lt: current, $gte: previous },
},
},
{
$sort: { createdAt: 1 },
},
{
$group: {
_id: null,
docs: {
$push: {
createdAt: "$createdAt",
totalPrice: "$totalPrice",
missing: false,
},
},
start: {
$first: {
$toInt: {
$divide: [
{
$subtract: [
{ $toLong: "$$NOW" },
{ $multiply: [24, 60, 60, 1000] },
],
},
1000,
],
},
},
},
end: {
$last: { $toInt: { $divide: [{ $toLong: "$$NOW" }, 1000] } },
},
},
},
{
$addFields: {
docs: {
$map: {
input: {
$range: [
{ $toInt: "$start" },
{ $add: [{ $toInt: "$end" }, 7200] }, // 2 hours range
7200,
],
},
as: "ts",
in: {
ts_exists: {
$filter: {
input: "$docs",
as: "d",
cond: {
$and: [
{
$gte: [
{
$toInt: {
$divide: [{ $toLong: "$$d.createdAt" }, 1000],
},
},
{ $subtract: ["$$ts", 7200] },
],
},
{
$lt: [
{
$toInt: {
$divide: [{ $toLong: "$$d.createdAt" }, 1000],
},
},
"$$ts",
],
},
],
},
},
},
ts: "$$ts",
},
},
},
},
},
{
$unwind: "$docs",
},
{
$project: {
_id: 0,
y: {
$reduce: {
input: "$docs.ts_exists",
initialValue: 0,
in: { $add: ["$$value", "$$this.totalPrice"] },
},
},
x: {
$dateToString: {
format: "%Y-%m-%d %H:%M",
date: { $toDate: { $multiply: ["$docs.ts", 1000] } },
},
},
},
},
]);
const firstDeleted = order.shift(); // the first always send 0, therefore I delete it
res.send(order);
} catch (error) {
res.send(error);
}
});
Returns data every 2 hours, if there is no quantity put it 0
[
{
"y": 0,
"x": "2020-09-15 18:24"
},
{
"y": 0,
"x": "2020-09-15 20:24"
},
{
"y": 0,
"x": "2020-09-15 22:24"
},
{
"y": 0,
"x": "2020-09-16 00:24"
},
{
"y": 0,
"x": "2020-09-16 02:24"
},
{
"y": 0,
"x": "2020-09-16 04:24"
},
{
"y": 0,
"x": "2020-09-16 06:24"
},
{
"y": 0,
"x": "2020-09-16 08:24"
},
{
"y": 0,
"x": "2020-09-16 10:24"
},
{
"y": 0,
"x": "2020-09-16 12:24"
},
{
"y": 0,
"x": "2020-09-16 14:24"
},
{
"y": 3,
"x": "2020-09-16 16:24"
}
]