MongoDB aggregation group average rating by last 7 days with empty values - node.js

Question

I'm attempting to query a collection and retrieve an average value for the each of the last 7 days excluding the current day. On some or all of the days there may not be an average.
Here's what I have so far:
var dateTill = moment({hour:0,minute:0}).subtract(1, 'days')._d
var dateSevenDaysAgo = moment({hour:0,minute:0}).subtract(7, 'days')._d;
Rating.aggregate([
{
$match:{
userTo:facebookId,
timestamp:{$gt:dateSevenDaysAgo,$lt:dateTill}
}
},
{
$group:{
_id:{day:{'$dayOfMonth':'$timestamp'}},
average:{$avg:'$rating'}
}
},
{
$sort:{
'_id.day':1
}
}
]
This gives me
[ { _id: { day: 20 }, average: 1 },
{ _id: { day: 22 }, average: 3 },
{ _id: { day: 24 }, average: 5 } ]
What I'm trying to get is something like:
[1,,3,,5,,]
Which represents the last 7 days of averages in order and has an empty element where there is no average for that day.
I could try and make a function that detects where the gaps are but this won't work when the averages are spread across two different months. e.g (July 28,29,30,31,Aug 1,2] - the days in august will be sorted to the front of the array I want.
Is there an easier way to do this?
Thanks!

Answer 1

People ask about "empty results" quite often, and the thinking usually comes from how they would have approached the problem with a SQL query.
But whilst it is "possible" to throw a set of "empty results" for items that do not contain a grouping key, it is a difficult process and much like the SQL approach people use, it's just throwing those values within the statement artificially and it really isn't a very performance driven alternative. Think "join" with a manufactured set of keys. Not efficient.
The smarter approach is to have those results ready in the client API directly, without sending to the server. Then the aggregation output can be "merged" with those results to create a complete set.
However you want to store the set to merge with is up to you, it just requires a basic "hash table" and lookups. But here is an example using nedb, which allows you to maintain the MongoDB set of thinking for query and updates:
var async = require('async'),
mongoose = require('mongoose'),
DataStore = require('nedb'),
Schema = mongoose.Schema,
db = new DataStore();
mongoose.connect('mongodb://localhost/test');
var Test = mongoose.model(
'Test',
new Schema({},{ strict: false }),
"testdata"
);
var testdata = [
{ "createDate": new Date("2015-07-20"), "value": 2 },
{ "createDate": new Date("2015-07-20"), "value": 4 },
{ "createDate": new Date("2015-07-22"), "value": 4 },
{ "createDate": new Date("2015-07-22"), "value": 6 },
{ "createDate": new Date("2015-07-24"), "value": 6 },
{ "createDate": new Date("2015-07-24"), "value": 8 }
];
var startDate = new Date("2015-07-20"),
endDate = new Date("2015-07-27"),
oneDay = 1000 * 60 * 60 * 24;
async.series(
[
function(callback) {
Test.remove({},callback);
},
function(callback) {
async.each(testdata,function(data,callback) {
Test.create(data,callback);
},callback);
},
function(callback) {
async.parallel(
[
function(callback) {
var tempDate = new Date( startDate.valueOf() );
async.whilst(
function() {
return tempDate.valueOf() <= endDate.valueOf();
},
function(callback) {
var day = tempDate.getUTCDate();
db.update(
{ "day": day },
{ "$inc": { "average": 0 } },
{ "upsert": true },
function(err) {
tempDate = new Date(
tempDate.valueOf() + oneDay
);
callback(err);
}
);
},
callback
);
},
function(callback) {
Test.aggregate(
[
{ "$match": {
"createDate": {
"$gte": startDate,
"$lt": new Date( endDate.valueOf() + oneDay )
}
}},
{ "$group": {
"_id": { "$dayOfMonth": "$createDate" },
"average": { "$avg": "$value" }
}}
],
function(err,results) {
if (err) callback(err);
async.each(results,function(result,callback) {
db.update(
{ "day": result._id },
{ "$inc": { "average": result.average } },
{ "upsert": true },
callback
)
},callback);
}
);
}
],
callback
);
}
],
function(err) {
if (err) throw err;
db.find({},{ "_id": 0 }).sort({ "day": 1 }).exec(function(err,result) {
console.log(result);
mongoose.disconnect();
});
}
);
Which gives this output:
[ { day: 20, average: 3 },
{ day: 21, average: 0 },
{ day: 22, average: 5 },
{ day: 23, average: 0 },
{ day: 24, average: 7 },
{ day: 25, average: 0 },
{ day: 26, average: 0 },
{ day: 27, average: 0 } ]
In short, a "datastore" is created with nedb, which basically acts the same as any MongoDB collection ( with stripped down functionality ). You then insert your range of "keys" expected and default values for any of the results.
Then running your aggregation statement, which is only going to return the keys that exist in the queried collection, you simply "update" the created datastore at the same key with the aggregated values.
To make that a bit more efficient, I am running both the empty result "creation" and the "aggregation" operations in parallel, utilizing "upsert" functionallity and the $inc operator for the values. These will not conflict, and that means the creation can happen at the same time as the aggregation is running, so no delays.
This is very simple to integrate into your API, so you can have all the keys you want, including those with no data for aggregation in the collection for output.
The same approach adapts well to using another actual collection on your MongoDB server for very large result sets. But if they are very large, then you should be pre-aggregating results anyway, and just using standard queries to sample.