Related
I have to generate this report.
In my MongoDB database I have a collection of orders like this:
[
{
_id: "mongoId", // 5f5ea6276ba53b06944de28c
createdAt: "2020-09-15T23:07:19.370Z",
totalPrice: 34, // is calculated from the client (quantity * price)
orderDetail: [
{
_id: "product-A-Id", // 5f5ea403e91ed91a44b62c92
quantity: 4,
price: 5.5,
},
{
_id: "product-B-Id",
quantity: 1,
price: 3.5,
},
{
_id: "product-C-Id",
quantity: 1,
price: 8.5,
},
],
},
{
_id: "mongoId",
createdAt: "2020-09-15T23:08:20.370Z",
totalPrice: 15.5,
orderDetail: [
{
_id: "product-C-Id",
quantity: 3,
price: 3,
},
{
_id: "product-D-Id",
quantity: 1,
price: 6.5,
},
],
},
{
_id: "mongoId",
createdAt: "2020-09-15T23:09:25.370Z",
totalPrice: 22.5,
orderDetail: [
{
_id: "product-D-Id",
quantity: 5,
price: 4.5,
},
],
},
]
To make this I have to generate time series data each two hours from timestamp now (in every request), the example of response desired is this:
[
{
id: "sales",
data: [
{
x: "00:00",
y: 150,
},
{
x: "22:00",
y: 100,
},
{
x: "20:00",
y: 150,
},
{
x: "18:00",
y: 50,
},
{
x: "16:00",
y: 100,
},
],
},
]
Using nodejs and express like framework I could generate sales of the last 2 hours:
const valueDateRange = 2 * 60 * 60 * 1000; // 2 hours
const currentPeriod = new Date(new Date().getTime() - valueDateRange);
// The last 2 hours sales
const calculateTotalSales = await Order.aggregate([
{
$match: { createdAt: { $gte: currentPeriod } },
},
{
$group: { _id: null, TotalAmount: { $sum: "$totalPrice" } },
},
]);
But now how to generate the time series data each 2 hours, so much thanks for the attention
The cleanest way I've found is to rework each docs date using the modulo operator to group them by hour blocks. You can easily change if you need bigger blocks in the future.
https://mongoplayground.net/p/aYAJKL_5dMD (I added extra sample data)
db.orders.aggregate([
{$addFields: {
date: {
$let: {
vars: {
hour: {$hour: '$createdAt'},
remainder: {$mod: [
{$hour: '$createdAt'},
2 // Two hour blocks, can be 2,3,4,6,8,12
]},
},
in: {
$dateFromParts: {
year: {$year: '$createdAt'},
month: {$month: '$createdAt'},
day: {$dayOfMonth: '$createdAt'},
hour: {$subtract: ['$$hour', '$$remainder']}
}
}
}
}
}},
{$group: {
_id: '$date',
x: {$last: '$date'},
y: {$sum: '$totalPrice'}
}}
]);
Update:
After reading your question again, I think your looking for total per hour regardless of the day. You can do so like this:
https://mongoplayground.net/p/cpW9JKllDIN
const totals = await db.orders.aggregate([
{$addFields: {
hour: {
$let: {
vars: {
hour: {$hour: '$createdAt'},
remainder: {$mod: [
{$hour: '$createdAt'},
2 // Two hour blocks, can be 2,3,4,6,8,12
]},
},
in: {$subtract: ['$$hour', '$$remainder']}
}
}
}},
{$group: {
_id: '$hour',
x: {$last: '$hour'},
y: {$sum: '$totalPrice'}
}}
])
Then to include hours that have no sales you can map an array:
let points = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22].map(x => {
let total = totals.find(t => t.x === x);
return {
x: `${x < 10 ? `0${x}` : x}:00`,
y: total ? total.y : 0
};
});
After trying many times, I ended up with this code, I hope it helps someone in the future, to prove it, make sure you have the updated data at the time you make the request, otherwise it would result 0 because compare the dates with the current moment.
Mongo Playgournd
node.js: v12.5.0
express: "^4.17.1"
Mongodb version v4.2.3
router.get("/orders", async (req, res) => {
let valueDateRange = 24 * 60 * 60 * 1000; // 24 hours
const current = new Date();
const previous = new Date(new Date().getTime() - valueDateRange);
try {
const order = await Order.aggregate([
{
$match: {
createdAt: { $lt: current, $gte: previous },
},
},
{
$sort: { createdAt: 1 },
},
{
$group: {
_id: null,
docs: {
$push: {
createdAt: "$createdAt",
totalPrice: "$totalPrice",
missing: false,
},
},
start: {
$first: {
$toInt: {
$divide: [
{
$subtract: [
{ $toLong: "$$NOW" },
{ $multiply: [24, 60, 60, 1000] },
],
},
1000,
],
},
},
},
end: {
$last: { $toInt: { $divide: [{ $toLong: "$$NOW" }, 1000] } },
},
},
},
{
$addFields: {
docs: {
$map: {
input: {
$range: [
{ $toInt: "$start" },
{ $add: [{ $toInt: "$end" }, 7200] }, // 2 hours range
7200,
],
},
as: "ts",
in: {
ts_exists: {
$filter: {
input: "$docs",
as: "d",
cond: {
$and: [
{
$gte: [
{
$toInt: {
$divide: [{ $toLong: "$$d.createdAt" }, 1000],
},
},
{ $subtract: ["$$ts", 7200] },
],
},
{
$lt: [
{
$toInt: {
$divide: [{ $toLong: "$$d.createdAt" }, 1000],
},
},
"$$ts",
],
},
],
},
},
},
ts: "$$ts",
},
},
},
},
},
{
$unwind: "$docs",
},
{
$project: {
_id: 0,
y: {
$reduce: {
input: "$docs.ts_exists",
initialValue: 0,
in: { $add: ["$$value", "$$this.totalPrice"] },
},
},
x: {
$dateToString: {
format: "%Y-%m-%d %H:%M",
date: { $toDate: { $multiply: ["$docs.ts", 1000] } },
},
},
},
},
]);
const firstDeleted = order.shift(); // the first always send 0, therefore I delete it
res.send(order);
} catch (error) {
res.send(error);
}
});
Returns data every 2 hours, if there is no quantity put it 0
[
{
"y": 0,
"x": "2020-09-15 18:24"
},
{
"y": 0,
"x": "2020-09-15 20:24"
},
{
"y": 0,
"x": "2020-09-15 22:24"
},
{
"y": 0,
"x": "2020-09-16 00:24"
},
{
"y": 0,
"x": "2020-09-16 02:24"
},
{
"y": 0,
"x": "2020-09-16 04:24"
},
{
"y": 0,
"x": "2020-09-16 06:24"
},
{
"y": 0,
"x": "2020-09-16 08:24"
},
{
"y": 0,
"x": "2020-09-16 10:24"
},
{
"y": 0,
"x": "2020-09-16 12:24"
},
{
"y": 0,
"x": "2020-09-16 14:24"
},
{
"y": 3,
"x": "2020-09-16 16:24"
}
]
I've been working on a small project that takes MQTT data from sensors and stores it in a MongoDB database. I'm working with nodeJS and mongoose. These are my schemas.
export const SensorSchema = new mongoose.Schema({
name: { type: String, required: true, unique: true },
location: { type: String, required: true },
type: { type: String, required: true },
unit: { type: String, required: true },
measurements: { type: [MeasurementSchema] }
},
{
toObject: { virtuals: true },
toJSON: { virtuals: true }
});
export const MeasurementSchema = new mongoose.Schema({
value: {type: Number, required: true},
time: {type: Date, required: true}
});
First I wrote a function that retrieves all measurements that were made in between two timestamps.
const values = Sensor.aggregate([
{ $match: Sensor.getValuesFromPath(sensorPath) },
{ $unwind: "$measurements"},
{ $match: { "measurements.time": { $gte: startTime, $lte: endTime} }},
{ $replaceRoot: { newRoot: "$measurements" } },
{ $project: { _id: 0}},
{ $sort: {time: 1}}
]).exec();
In order to draw a graph in the UI, I need to somehow sort and then limit the data that gets sent to the client. I want to send every Nth Value in a certain interval to ensure that the data somewhat resembles the course of the data.
I would prefer a solution that doesn't fetch all the data from the database.
How would I go about doing this on the db? Can I somehow access the positional index of an element after sorting it? Is $arrayElemAt or $elemMatch the solution?
Befure you run $unwind you can use $filter to apply start/end Date filtering. This will allow you to process measurements as an array. In the next step you can get every N-th element by using $range to define a list of indexes and $arrayElemAt to retrieve elements from these indexes:
const values = Sensor.aggregate([
{ $match: Sensor.getValuesFromPath(sensorPath) },
{ $addFields: {
measurements: {
$filter: {
input: "$measurements",
cond: { $and: [
{ $gte: [ "$$this.time", startTime ] },
{ $lte: [ "$$this.time", endTime ] }
]
}
}
}
} },
{ $addFields: {
measurements: {
$map: {
input: input: { $range: [ 0, { $size: "$measurements" }, N ] },
as: "index",
in: { $arrayElemAt: [ "$measurements", "$$index" ] }
}
}
} },
{ $unwind: "$measurements" },
{ $replaceRoot: { newRoot: "$measurements" } },
{ $project: { _id: 0}},
{ $sort: {time: 1}}
]).exec();
The following aggregation (i) retrieves all measurements that were made in between two timestamps, (ii) sorts by timestamp for each sensor, and (iii) gets every Nth value (specified by the variable EVERY_N).
Sample documents (with some arbitrary data for testing):
{
name: "s-1",
location: "123",
type: "456",
measurements: [ { time: 2, value: 12 }, { time: 3, value: 13 },
{ time: 4, value: 15 }, { time: 5, value: 22 },
{ time: 6, value: 34 }, { time: 7, value: 9 },
{ time: 8, value: 5 }, { time: 9, value: 1 },
]
},
{
name: "s-2",
location: "789",
type: "900",
measurements: [ { time: 1, value: 31 }, { time: 3, value: 32 },
{ time: 4, value: 35 }, { time: 6, value: 39 },
{ time: 7, value: 6}, { time: 8, value: 70 },
{ time: 9, value: 74 }, { time: 10, value: 82 }
]
}
The aggregation:
var startTime = 3, endTime = 10
var EVERY_N = 2 // value can be 3, etc.
db.collection.aggregate( [
{
$unwind: "$measurements"
},
{
$match: {
"measurements.time": { $gte: startTime, $lte: endTime }
}
},
{
$sort: { name: 1, "measurements.time": 1 }
},
{
$group: {
_id: "$name",
measurements: { $push: "$measurements" },
doc: { $first: "$$ROOT" }
}
},
{
$addFields: {
"doc.measurements": "$measurements"
}
},
{
$replaceRoot: { newRoot: "$doc" }
},
{
$addFields: {
measurements: {
$reduce: {
input: { $range: [ 0, { $size: "$measurements" } ] },
initialValue: [ ],
in: { $cond: [ { $eq: [ { $mod: [ "$$this", EVERY_N ] }, 0 ] },
{ $concatArrays: [ "$$value", [ { $arrayElemAt: [ "$measurements", "$$this" ] } ] ] },
"$$value"
]
}
}
}
}
}
] )
how you doing?
I have a trouble making a aggregation in my project, my aggregation result is different in Robo3T and Node.
db.getCollection('companies').aggregate([
{ '$match': { _id: { '$eq': ObjectId("5e30a4fe11e6e80d7fb544a4")} } },
{ $unwind: '$jobVacancies' },
{
$project: {
jobVacancies: {
_id: 1,
name: 1,
city: 1,
openingDate: 1,
closingDate: 1,
createdAt: 1,
quantity: 1,
steps: {
$filter: {
input: '$jobVacancies.steps',
as: 'step',
cond: {
$and: [
{ $eq: ['$$step.order', 0] },
{ $ne: ['$$step.users', undefined] },
{ $ne: ['$$step.users', null] },
{ $ne: ['$$step.users', []] },
],
},
},
},
},
},
},
{ $match: { 'jobVacancies.steps': { $ne: [] } } },
])
In Robo3T this is returning 1 object, but in Node (the same aggregation) is resulting 6 objects. Can you help me? Thank you
EDIT
Nodejs:
The first match create the ObjectId match for company in context of GraphQL based on my filter.
const companies = await this.MongoClient.db
.collection('companies')
.aggregate([
{
$match: await this.getFilterObject(
filters.filter(f => !f.field.includes('$$jobVacancy') && !f.field.includes('StepOrder')),
),
},
{ $unwind: '$jobVacancies' },
{
$project: {
jobVacancies: {
_id: 1,
name: 1,
city: 1,
openingDate: 1,
closingDate: 1,
createdAt: 1,
quantity: 1,
steps: {
$filter: {
input: '$jobVacancies.steps',
as: 'step',
cond: {
$and: [
{ $eq: ['$$step.order', order] },
{ $ne: ['$$step.users', undefined] },
{ $ne: ['$$step.users', null] },
{ $ne: ['$$step.users', []] },
],
},
},
},
},
},
},
{ $match: { 'jobVacancies.steps': { $ne: [] } } },
])
.toArray();
EDIT 3
This is the result of console.dir (with {depth:null}) of the pipeline
[
{
'$match': {
_id: {
'$eq': ObjectID {
_bsontype: 'ObjectID',
id: Buffer [Uint8Array] [
94, 48, 164, 254, 17,
230, 232, 13, 127, 181,
68, 164
]
}
}
}
},
{ '$unwind': '$jobVacancies' },
{
'$project': {
jobVacancies: {
_id: 1,
name: 1,
city: 1,
openingDate: 1,
closingDate: 1,
createdAt: 1,
quantity: 1,
steps: {
'$filter': {
input: '$jobVacancies.steps',
as: 'step',
cond: {
'$and': [
{ '$eq': [ '$$step.order', 0 ] },
{ '$ne': [ '$$step.users', undefined ] },
{ '$ne': [ '$$step.users', null ] },
{ '$ne': [ '$$step.users', [] ] }
]
}
}
}
}
}
},
{ '$match': { 'jobVacancies.steps': { '$ne': [] } } }
]
I think i found the solution, the document is created with properties:
jobVacancies: {
steps: {
users: []
}
}
But sometimes users array is undefined in mongodb, so I verify with
{ '$ne': [ '$$step.users', undefined ] }
I think JS undefined is different then mongodb undefined, so I initialized all steps with an empty array of users, and removed this verification and worked! –
I have this schema in Mongoose:
var CoinAmountSchema = new Schema(
{
user: [{ type: Schema.ObjectId, ref: 'User' }],
coinAmounts: [{
_id: false,
coinID: { type: Number, ref: 'Coin' },
amount: Number
}]
})
I am writing this query, that checks the userID and coinID and should update the amount of only that coinID's amount.
exports.coin_amount_update = [
(req, res, next) => {
CoinAmount.update({
"user": req.params.userId,
"coinAmounts.coinID": req.params.coinId
},
{
'$set': {
'coinAmounts.$.amount': req.body.amount
}
},
function (err, model) {
if (err) {
console.log(err)
return res.send(err)
}
return res.json(model)
})
}]
But like this, it only updates the first coin's in the array amount. BUT, if I delete the line "user": req.params.userId, it would find and update the right coin. I need to check for a user as well though, so how can I make it work?
Is there something wrong with the query or the way the data is structured?
EDIT: I send a request in React-native:
fetch(`${apiBaseURL}/users/${getState().user._id}/coins/${id}/update`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
amount: getState().coins[id].amount
}),
})
If the request is /users/:userID/coins/0/update (with amount: 1)
then the result will be
{ _id: 5a579d0d44e7390ba3029327,
__v: 0,
coinAmounts:
[ { coinID: 0, amount: 1 },
{ coinID: 1, amount: 0 },
{ coinID: 2, amount: 0 },
{ coinID: 3, amount: 0 },
{ coinID: 4, amount: 0 } ],
user: [ 5a579d0d44e7390ba3029326 ] }
The same result if the request is /users/:userID/coins/1/update with the same amount.
But if as mentioned before, I remove the check for userID, the request /users/:userID/coins/1/update would produce this:
{ _id: 5a579d0d44e7390ba3029327,
__v: 0,
coinAmounts:
[ { coinID: 0, amount: 0 },
{ coinID: 1, amount: 1 },
{ coinID: 2, amount: 0 },
{ coinID: 3, amount: 0 },
{ coinID: 4, amount: 0 } ],
user: [ 5a579d0d44e7390ba3029326 ] }
Hope I was clear.
It looks like a bug when using two arrays in find with $ positional update, it gets matching index of user for $ positional update
tried below workarounds, both updates correct coinID
workaround-1, using arrayFilters
db.coins.update(
{ "user" : "5a579d0d44e7390ba3029326" }, //user
{
$set: { "coinAmounts.$[elem].amount" : 1 } //update
},
{
multi: false,
arrayFilters: [
{ "elem.coinID": 2 } //coinID
]
}
)
workaround-2, using elemMatch for user array
db.coins.update(
{
"coinAmounts.coinID" : 1, //coinID
"user" : { $elemMatch : { $eq : "5a579d0d44e7390ba3029326" } } //user
},
{ $set : { "coinAmounts.$.amount" : 1 } } //update
)
Let's say I have an array of Movie genres like so:
[
{ id: 28, name: 'Action' },
{ id: 12, name: 'Adventure' },
{ id: 16, name: 'Animation' },
{ id: 35, name: 'Comedy' },
{ id: 80, name: 'Crime' },
{ id: 99, name: 'Documentary' },
{ id: 18, name: 'Drama' },
{ id: 10751, name: 'Family' },
{ id: 14, name: 'Fantasy' },
{ id: 10769, name: 'Foreign' },
{ id: 36, name: 'History' },
{ id: 27, name: 'Horror' },
{ id: 10402, name: 'Music' },
{ id: 9648, name: 'Mystery' },
{ id: 10749, name: 'Romance' },
{ id: 878, name: 'Science Fiction' },
{ id: 10770, name: 'TV Movie' },
{ id: 53, name: 'Thriller' },
{ id: 10752, name: 'War' },
{ id: 37, name: 'Western' }
]
and I have a connection to a MongoDB (v3.2) instance: db, and I'm using the standard mongodb Node.js driver (const mongodb = require('mongodb').MongoClient).
What I want to be able to do is one bulk upsert operation onto a collection, say genres, where the _id field maps to the id field of our genre objects.
Now, I know I could loop through each item in the array, and do a simple upsert:
for (let i = 0; i < genres.length; i++) {
await db.collection('genres').update(
{ _id: genres[i].id },
genres[i],
{ upsert: true }
);
}
But this feels wasteful and wrong.
Is there an easier way to do what should be a relatively simple task?
Thanks
Use the bulkWrite API to carry out the updates:
var bulkUpdateOps = genres.map(function(doc) {
return {
"updateOne": {
"filter": { "_id": doc.id },
"update": { "$set": { "name": doc.name } },
"upsert": true
}
};
});
db.collection('genres').bulkWrite(bulkUpdateOps, function(err, r) {
// do something with result
})
If you're dealing with larger arrays i.e. > 1000 then consider sending the writes to the server in batches of 500 which gives you a better performance as you are not sending every request to the server, just once in every 500 requests:
var bulkUpdateOps = [],
counter = 0;
genres.forEach(function(doc) {
bulkUpdateOps.push({
"updateOne": {
"filter": { "_id": doc.id },
"update": { "$set": { "name": doc.name } },
"upsert": true
}
});
counter++;
if (counter % 500 == 0) {
db.collection('genres').bulkWrite(bulkUpdateOps, function(err, r) {
// do something with result
});
bulkUpdateOps = [];
}
})
if (counter % 500 != 0) {
db.collection('genres').bulkWrite(bulkUpdateOps, function(err, r) {
// do something with the result
});
}
I would try:
db.collection('genres').update(genres, {upsert: true, multi: true});
Note: untested code...
UPDATE: to remap id field to _id:
var _genres = genres.map(function(genre) {
return { _id: genre.id, name: genre.name };
});
db.collection('genres').update(_genres, {upsert: true, multi: true});