MongoDB aggregate lookup match not working the same without lookup - node.js

Question

Can you please help? I'm trying to aggregate data over the past 12 months by both ALL publication data specific to a certain publisher and per publication to return a yearly graph data analysis based on the subscription type.
Here's a snapshot of the Subscriber model:
const SubscriberSchema = new Schema({
publication: { type: Schema.Types.ObjectId, ref: "publicationcollection" },
subType: { type: String }, // Print, Digital, Bundle
subStartDate: { type: Date },
subEndDate: { type: Date },
});
Here's some data for the reader (subscriber) collection:
{
_id: ObjectId("5dc14d3fc86c165ed48b6872"),
publication: ObjectId("5d89db9d82273f1d18970deb"),
subStartDate: "2019-11-20T00:00:00.000Z",
subtype: "print"
},
{
_id: ObjectId("5dc14d3fc86c165ed48b6871"),
publication: ObjectId("5d89db9d82273f1d18970deb"),
subStartDate: "2019-11-19T00:00:00.000Z",
subtype: "print"
},
{
_id: ObjectId("5dc14d3fc86c165ed48b6870"),
publication: ObjectId("5d89db9d82273f1d18970deb"),
subStartDate: "2019-11-18T00:00:00.000Z",
subtype: "digital"
},
{
_id: ObjectId("5dc14d3fc86c165ed48b6869"),
publication: ObjectId("5d8b36c3148c1e5aec64662c"),
subStartDate: "2019-11-19T00:00:00.000Z",
subtype: "print"
}
The publication model has plenty of fields but the _id and user fields are the only point of reference in the following queries.
Here's some data for the publication collection:
// Should use
{ "_id": {
"$oid": "5d8b36c3148c1e5aec64662c"
},
"user": {
"$oid": "5d24bbd89f09024590db9dcd"
},
"isDeleted": false
},
// Should use
{ "_id": {
"$oid": "5d89db9d82273f1d18970deb"
},
"user": {
"$oid": "5d24bbd89f09024590db9dcd"
},
"isDeleted": false
},
// Shouldn't use as deleted === true
{ "_id": {
"$oid": "5d89db9d82273f1d18970dec"
},
"user": {
"$oid": "5d24bbd89f09024590db9dcd"
},
"isDeleted": true
},
// Shouldn't use as different user ID
{ "_id": {
"$oid": "5d89db9d82273f1d18970dfc"
},
"user": {
"$oid": "5d24bbd89f09024590db9efd"
},
"isDeleted": true
}
When I do a lookup on a publication ID with the following, I'm getting perfect results:
Subscriber.aggregate([
{
$match: {
$and: [
{ 'publication': mongoose.Types.ObjectId(req.params.id) },
],
"$expr": { "$eq": [{ "$year": "$subStartDate" }, new Date().getFullYear()] }
}
},
{
/* group by year and month of the subscription event */
$group: {
_id: { year: { $year: "$subStartDate" }, month: { $month: "$subStartDate" }, subType: "$subType" },
count: { $sum: 1 }
},
},
{
/* sort descending (latest subscriptions first) */
$sort: {
'_id.year': -1,
'_id.month': -1
}
},
{
$limit: 100,
},
])
However, when I want to receive data from the readercollections (Subscriber Model) for ALL year data, I'm not getting the desired results (if any) from all of the things I'm trying (I'm posting the best attempt result below):
Publication.aggregate([
{
$match:
{
user: mongoose.Types.ObjectId(id),
isDeleted: false
}
},
{
$project: {
_id: 1,
}
},
{
$lookup: {
from: "readercollections",
let: { "id": "$_id" },
pipeline: [
{
$match:
{
$expr: {
$and: [
{ $eq: ["$publication", "$$id"] },
{ "$eq": [{ "$year": "$subStartDate" }, new Date().getFullYear()] }
],
}
}
},
{ $project: { subStartDate: 1, subType: 1 } }
],
as: "founditem"
}
},
// {
// /* group by year and month of the subscription event */
// $group: {
// _id: { year: { $year: "$founditem.subStartDate" }, month: { $month: "$foundtitem.subStartDate" }, subType: "$founditem.subType" },
// count: { $sum: 1 }
// },
// },
// {
// /* sort descending (latest subscriptions first) */
// $sort: {
// '_id.year': -1,
// '_id.month': -1
// }
// },
], function (err, result) {
if (err) {
console.log(err);
} else {
res.json(result);
}
})
Which returns the desired data without the $group (commented out) but I need the $group to work or I'm going to have to map a dynamic array based on month and subtype which is completely inefficient.
When I'm diagnosing, it looks like this $group is the issue but I can't see how to fix as it works in the singular $year/$month group. So I tried the following:
{
/* group by year and month of the subscription event */
$group: {
_id: { year: { $year: "$subStartDate" }, month: { $month: "$subStartDate" }, subType: "$founditem.subType" },
count: { $sum: 1 }
},
},
And it returned the $founditem.subType fine, but any count or attempt to get $year or $month of the $founditem.subStartDate gave a BSON error.
The output from the single publication ID lookup in the reader collection call that works (and is plugging into the line graph perfectly) is:
[
{
"_id": {
"year": 2019,
"month": 11,
"subType": "digital"
},
"count": 1
},
{
"_id": {
"year": 2019,
"month": 11,
"subType": "print"
},
"count": 3
}
]
This is the output I'd like for ALL publications rather than just a single lookup of a publication ID within the reader collection.
Thank you for any assistance and please let me know if you need more details!!