Mongo occurance count by column - node.js

I have a usecase to find the count of different statues like active, in-active, in-progress, etc,
the documents look like this -
{
"id": "1"
"status": "active"
},
{
"id": "2"
"status": "active"
},
{
"id": "3"
"status": "in-active"
},
{
"id": "4"
"status": "in-progress"
}
I needed output like -
{
"active": 2,
"in-active": 1,
"in-progress": 1
}
I am referring this answer but, not able to get the expected output -
Mongo count occurrences of each value for a set of documents
My code is as follows -
const mongoClient = require('mongodb').MongoClient;
const test = async () => {
const mongoUri = "mongodb://localhost:27017/";
const dbClientConnection = await mongoClient.connect(mongoUri, {
useNewUrlParser: true,
useUnifiedTopology: true
});
const db = await dbClientConnection.db("database name here");
const collection = await db.collection("collection name here");
let result = await collection.aggregate([
{
$group: {
_id: "$status",
sum: { $sum: 1 }
}
},
{
$group: {
_id: null,
status: {
$push: { k: "$_id", v: "$sum" }
}
}
},
{
$replaceRoot: {
newRoot: { $arrayToObject: "$status" }
}
}
])
console.log("result => ", result);
return result;
}
test();

The first stage is correct
$group by null and construct the array of key and value format
$arrayToObject convert above converted key-value pair array to an object
$replaceRoot to replace above object to root
let result = await collection.aggregate([
{
$group: {
_id: "$status",
sum: { $sum: 1 }
}
},
{
$group: {
_id: null,
status: {
$push: { k: "$_id", v: "$sum" }
}
}
},
{
$replaceRoot: {
newRoot: { $arrayToObject: "$status" }
}
}
])
Playground

Related

Nodejs Mongoose | Fetch the array of values for a given field

From the query below
let fields = { 'local.email': 1 };
UserModel.find({ '_id': { $in: userIds } }).select(fields).setOptions({ lean: true });
Result which we get is
[
{
"_id": "54bf2d7415eaaa570c9ed5a0",
"local": {
"email": "neo#q.com"
}
},
{
"_id": "54bfb753e4c9406112267056",
"local": {
"email": "test#q.com"
}
}
]
Is is possible to modify query itself to get below result
["neo#q.com", "test#q.com"]
Thanks in advance
You could use aggregate to return a list of objects with the emails and the map them to an array of strings:
const emailObjs = await UserModel.aggregate([
{
$match: {
_id: {
$in: userIds
}
}
},
{
$project: {
"_id": 0,
"email": "$local.email"
}
}
]);
const emails = emailObjs.map(obj => obj.email)
Link to playground for the query.

MongoServerError: PlanExecutor error during aggregation how to fix?

in the shopping cart project, am trying to get the total amount in the cart but my aggregation method has some problems , how to fix this ?
error
callback(new error_1.MongoServerError(document));
^
MongoServerError: PlanExecutor error during aggregation :: caused by :: Failed to parse number '1,39,999' in $convert with no onError value: Did not consume whole string.
user_helpers.js this the aggregation
getTotalAmount:(userId)=>{
return new Promise(async(resolve,reject)=>{
let total=await db.get().collection(collection.CART_COLLECTION).aggregate([
{
$match:{user:objectId(userId)}
},
{
$unwind:'$products'
},{
$project:{
item:'$products.item',
quantity:'$products.quantity',
}
},
{
$lookup:{
from:collection.PRODUCT_COLLECTION,
localField:'item',
foreignField:'_id',
as:'product'
}
},
{
$project:{
item:1,quantity:1,product:{$arrayElemAt:['$product',0]}
}
},
{
$group:{
_id:null,
total:{$sum:{$multiply:[{ $toInt: '$quantity'},{ $toInt: '$product.Price' }]}} // my assumption , here comes the error
}
}
]).toArray()
resolve(total.length > 0 ? total[0].total: 0) // or here
})
}
user.js
// GET: view shopping cart contents
router.get('/cart',middleware.verifyLogin,async(req,res,next)=>{
try{
let user=req.session.user._id;
let products =await userHelpers.getCartProducts(req.session.user._id)
let totalValue=0
if(products.length>0){
totalValue=await userHelpers.getTotalAmount(req.session.user._id)
let proId=req.params.id
console.log('proId>>>>>>>>>>',proId);
}
console.log('products>',products)
console.log("user...",user);
res.render('user/cart',{products,user,totalValue,});
}catch (error) {
console.log(error);
}
})
It comes when you click the cart button error is getting ,
how to fix this ?
The error message says itself, your database contains strings like 1,39,999. When MongoDB tries to convert these to an integer, it fails because of the commas. So, before converting the string to an integer, you should remove all the commas and other non-numeric characters from the string. Like this:
db.collection.aggregate([
{
$match: {
user: ObjectId("userId433456666666666666")
}
},
{
$unwind: "$products"
},
{
$project: {
item: "$products.item",
quantity: "$products.quantity",
}
},
{
$lookup: {
from: "collection.PRODUCT_COLLECTION",
localField: "item",
foreignField: "_id",
as: "product"
}
},
{
$project: {
item: 1,
quantity: 1,
product: {
$arrayElemAt: [
"$product",
0
]
}
}
},
{
$group: {
_id: null,
total: {
$sum: {
$multiply: [
{
$toInt: '$quantity'
},
{
$toInt: {
"$replaceAll": {
"input": "$product.Price",
"find": ",",
"replacement": ""
}
}
}
]
}
}
}
}
]).toArray()
Here we are using $replaceAll to replace the commas with an empty string.

how to group by in mongoose

Let's say I have a Collection with a field name and only "Sally" and "Bob" names exist. I want to group my results by this value. I'm currently getting my results and then using underscore to perform the group. But I should be able to do this with an aggregate.
The following is my code
const names = ['Bob', 'Sally'];
const docs = Collection.find({name: { $in: names}}, { fields: { name:1, age:1}, sort: { name:-1 } }).fetch();
//How can I do this part in the query above
const { Bob, Sally } = _.groupBy(docs, "name");
You can try,
$match your condition
$group by name and make array called fields and push name and age
$sort by _id means name
$replaceWith to replace object in root, $arrayToObject convert k, v format to object
const names = ['Bob', 'Sally'];
const docs = Collection.aggregate([
{ $match: { name: { $in: names } } },
{
$group: {
"_id": "$name",
fields: {
$push: {
name: "$name",
age: "$age"
}
}
}
},
{ $sort: { _id: -1 } },
{
$replaceWith: {
$arrayToObject: [
[
{
k: "$_id",
v: "$fields"
}
]
]
}
}
])
Playground
You can do something like this :
db.collection.aggregate([
{
$match: {
name: {
$in: names
}
}
},
{
$group: {
"_id": "$name",
age: {
$first: "$age"
}
}
},
{
$sort: {
name: -1
}
}
])
Try it here : MongoPlayground

Why am I getting a AggregationCursor as a result and not an average?

I'm querying my MongoDB database and don't understand why I am getting an aggregator cursor as a result when I expect to be returned a single number. Maybe I need to get something from the cursor object? Just can't figure out what.
module.exports = CalculateAvg = async collection => {
try {
// const count = await collection.countDocuments({ word: "Hello" });
// console.log(count) // logs 140, which shows that it is accessing the db correctly
const cursor = await collection.aggregate([
{ $match: { word: "Hello" } },
{
$group: {
_id: null,
mean: {
$avg: "$value" // in the dataset, each doc has a value field which equals a number
}
}
}
]);
console.log(cursor) // logs a large AggregationCursor object, rather than a number
} catch (err) {
console.log(err);
}
};
It's because aggregate return value is aggregateCursor, I recommend checking the Mongo's Nodejs driver types file whenever you're not sure whats the return value or the parameter value for any of these functions is.
You want to use cursor toArray like so:
const cursor = await collection.aggregate([
{ $match: { word: "Hello" } },
{
$group: {
_id: null,
mean: {
$avg: "$value" // in the dataset, each doc has a value field which equals a number
}
}
}
]).toArray();
You should use next() method... For Example
const pipeline = [{
$facet: {
total: [{
$count: 'createdAt'
}],
data: [{
$addFields: {
_id: '$_id'
}
}],
},
},
{
$unwind: '$total'
},
{
$project: {
data: {
$slice: ['$data', skip, {$ifNull: [limit,'$total.createdAt']} ]
},
meta: {
total: '$total.createdAt',
limit: {
$literal: limit
},
page: {
$literal: ((skip/limit) + 1)
},
pages: {
$ceil: {
$divide: ['$total.createdAt', limit]
}
}
}
}
}];
const document = await collection.aggregate(pipeline);
const yourData = await document.next();

Query for Matching Dates within Array

I have created a model schema with some nested fields in it, one of it is the Timestamp field:
{_id: Object_id,
name: string,
someArray: [{Timestamp: Date, otherFields: ...},{Timestamp: Date, otherFields...},...],
...,
}
the Timestamp has of type: Timestamp: Date
e.g (Timestamp: 2018-06-01T14:57:45.757647Z)
Now, I want to query only those documents from the array, which are between a start and end date that are received as parameters from a API url...
/link/Collection/:start.:end.:id
My router url (with the parameter strings as query) looks like this:
http://localhost:6000/link/Collection/2018-06-01T14:50:45.2018-06-01T15:17:45.29
My query function in mongoose / node (express) to retrieve the data looks like this:
exports.list_by_start_end_date_ID = function(req, res) {
console.log(req.body)
d_id=req.params.id;
start = req.params.start;
end = req.params.end;
console.log(d_id);
console.log(start)
console.log(end)
console.log(new Date(start));
console.log(new Date(end));
//SomeColl.findById(d_id, "myCollection").where("myCollection.Timestamp").gte(new Date(start)).lte(new Date(end))
SomeColl.findById(d_id, "myCollection",{"myCollection.Timestamp": {"$gte": new Date(start), "$lte": new Date(end)}})
.exec( function(err, fps) {
if (err)
res.send(err);
res.json(fps);
});
};
I get returned:
[{"Timestamp":"2018-06-01T14:57:45.757647Z"...},{"Timestamp":"2018-06-01T15:27:45.757647Z"...},{"Timestamp":"2018-06-01T15:12:45.757647Z"...}]
I don't get any error, I also can create new Date(start) from start and end parameters and it's correct, but as you can see, the document with 15:27 time shouldn't be returned...
I tried out both versions (also commented out version) of the query strings, and I also tried with the blank ISO Date format string that I passed as parameter (start / end) to the url.. but neither worked. How can I compare the dates in mongoose and get the correct documents passed back?
EDIT: I tried to find a workaround by ignoring db api operations, and just parsing the correct documents (subdocuments) of the array with javascript..:
myColl.findById(d_id)
.exec( function(err, fps) {
if (err) {
console.log(err);
res.send(err);
}
else {
//console.log(fps["someArray"])
laenge = fps["someArray"].length;
console.log(laenge);
startts = +new Date(start)
endts = +new Date(end)
TSarray = []
console.log(startts,endts)
for (let doc of fps["someArray"]) {
ts = +new Date(doc["Timestamp"])
//console.log(doc)
if ((ts >= startts) && (ts <= endts)){
TSarray.push(doc)
//console.log(TSarray)
}
}
res.json(TSarray)
//res.json(fps);
}
})//.then((res) => res.json())
};
However, when I want to get the results from the array, I get HTTP 304 error..
I did not find out yet, how to retrieve the corresponding subdocuments (based on a filter criteria) of an array field of one single document..
Do I have to use projection to get only the array field, and then use some filter criteria on that array to get the right subdocuments, or how does it generally work?
//EDIT2:
I tried with the mongoDB aggregation framework, but get returned []:
myColl.aggregate([{$match: {"id":d_id},
someArray: {
$filter: {
input: "$someArray",
as: "fp",
cond: {$and: [
{$gte: [ "$$fp.Timestamp", new Date(start)]},
{$lte: [ "$$fp.Timestamp", new Date(end)]}
]}
}
}
}
}]).exec( function(err, fps) {
if (err) {
console.log(err);
res.send(err);
}
else {
console.log(fps)
res.json(fps);
}
})}
;
This also does not work, is there anything wrong with that query? How can I specify a date range in mongoose with the filter criteria condition?
//EDIT3:
After 5 days of work, I finally managed to get the right documents returned, based on a timestamp. However, to get documents from 14:00:00 o'clock, I have to enter 16:00:00 as url parameter... I know it probably has something to do with UTC and timezones... my tz is Berlin, so I think its UTC +2 as MongoDB servers are in NY I think... How can I best accomodate to that problem?
Here is my function:
myColl.findById(d_id, "someArray")
.exec( function(err, fps) {
if (err) {
console.log(err);
res.send(err);
}
else {
startts = +new Date(start)
endts = +new Date(end)
TSarray = []
for (let doc of fps["Fahrplanabschnitte"]) {
ts = + new Date(doc["Timestamp"]
if ((ts >= startts) && (ts <= endts)){
TSarray.push(doc)
}
}
//for (let a of TSarray) {console.log(a)};
res.json(TSarray);
}
})
};
You're missing the $elemMatch operator on the basic query and the $filter you attempted with the aggregation framework actually has incorrect syntax.
So returning the document matching the dates being within that range in the array is:
// Simulating the date values
var start = new Date("2018-06-01"); // otherwise new Date(req.params.start)
var end = new Date("2018-07-01"); // otherwise new Date(req.params.end)
myColl.find({
"_id": req.params.id,
"someArray": {
"$elemMatch": { "$gte": start, "$lt": end }
}
}).then( doc => {
// do something with matched document
}).catch(e => { console.err(e); res.send(e); })
Filtering the actual array elements to be returned is:
// Simulating the date values
var start = new Date("2018-06-01");
var end = new Date("2018-07-01");
myColl.aggregate([
{ "$match": {
"_id": mongoose.Types.ObjectId(req.params.id),
"someArray": {
"$elemMatch": { "$gte": start, "$lt": end }
}
}},
{ "$project": {
"name": 1,
"someArray": {
"$filter": {
"input": "$someArray",
"cond": {
"$and": [
{ "$gte": [ "$$this.Timestamp", start ] }
{ "$lt": [ "$$this.Timestamp", end ] }
]
}
}
}
}}
]).then( docs => {
/* remember aggregate returns an array always, so if you expect only one
* then it's index 0
*
* But now the only items in 'someArray` are the matching ones, so you don't need
* the code you were writing to just pull out the matching ones
*/
console.log(docs[0].someArray);
}).catch(e => { console.err(e); res.send(e); })
The things to be aware of are that in the aggregate() you need to actually "cast" the ObjectId value, because Mongoose "autocasting" does not work here. Normally mongoose reads from the schema to determine how to cast the data, but since aggregation pipelines "change things" then this does not happen.
The $elemMatch is there because as the documentation says:
When specifying conditions on more than one field nested in an array of documents, you can specify the query such that either a single document meets these condition or any combination of documents (including a single document) in the array meets the conditions.
Use $elemMatch operator to specify multiple criteria on an array of embedded documents such that at least one embedded document satisfies all the specified criteria.
In short $gte and $lt are an AND condition and count as "two", therefore the simple "dot notation" form does not apply. It's also $lt and not $lte, since it makes more sense to be "less than" the "next day" rather than looking for equality up to the "last millisecond".
The $filter of course does exactly what it's name suggests and "filters" the actual array content so that only matching items are left behind.
Demonstration
Full demonstration listing creates two documents, one having only two array items which actually match the date range. The first query shows the correct document is matched with the range. The second shows the "filtering" of the array:
const { Schema, Types: { ObjectId } } = mongoose = require('mongoose');
const uri = 'mongodb://localhost/test';
mongoose.Promise = global.Promise;
mongoose.set('debug',true);
const subSchema = new Schema({
timestamp: Date,
other: String
});
const testSchema = new Schema({
name: String,
someArray: [subSchema]
});
const Test = mongoose.model('Test', testSchema, 'filtertest');
const log = data => console.log(JSON.stringify(data, undefined, 2));
const startDate = new Date("2018-06-01");
const endDate = new Date("2018-07-01");
(function() {
mongoose.connect(uri)
.then(conn =>
Promise.all(Object.entries(conn.models).map(([k,m]) => m.remove()))
)
.then(() =>
Test.insertMany([
{
_id: "5b1522f5cdac0b6da18f7618",
name: 'A',
someArray: [
{ timestamp: new Date("2018-06-01"), other: "C" },
{ timestamp: new Date("2018-07-04"), other: "D" },
{ timestamp: new Date("2018-06-10"), other: "E" }
]
},
{
_id: "5b1522f5cdac0b6da18f761c",
name: 'B',
someArray: [
{ timestamp: new Date("2018-07-04"), other: "D" },
]
}
])
)
.then(() =>
Test.find({
"someArray": {
"$elemMatch": {
"timestamp": { "$gte": startDate, "$lt": endDate }
}
}
}).then(docs => log({ docs }))
)
.then(() =>
Test.aggregate([
{ "$match": {
"_id": ObjectId("5b1522f5cdac0b6da18f7618"),
"someArray": {
"$elemMatch": {
"timestamp": { "$gte": startDate, "$lt": endDate }
}
}
}},
{ "$addFields": {
"someArray": {
"$filter": {
"input": "$someArray",
"cond": {
"$and": [
{ "$gte": [ "$$this.timestamp", startDate ] },
{ "$lt": [ "$$this.timestamp", endDate ] }
]
}
}
}
}}
]).then( filtered => log({ filtered }))
)
.catch(e => console.error(e))
.then(() => mongoose.disconnect());
})()
Or a bit more modern with async/await syntax:
const { Schema, Types: { ObjectId } } = mongoose = require('mongoose');
const uri = 'mongodb://localhost/test';
mongoose.Promise = global.Promise;
mongoose.set('debug',true);
const subSchema = new Schema({
timestamp: Date,
other: String
});
const testSchema = new Schema({
name: String,
someArray: [subSchema]
});
const Test = mongoose.model('Test', testSchema, 'filtertest');
const log = data => console.log(JSON.stringify(data, undefined, 2));
(async function() {
try {
const startDate = new Date("2018-06-01");
const endDate = new Date("2018-07-01");
const conn = await mongoose.connect(uri);
// Clean collections
await Promise.all(Object.entries(conn.models).map(([k,m]) => m.remove()));
// Create test items
await Test.insertMany([
{
_id: "5b1522f5cdac0b6da18f7618",
name: 'A',
someArray: [
{ timestamp: new Date("2018-06-01"), other: "C" },
{ timestamp: new Date("2018-07-04"), other: "D" },
{ timestamp: new Date("2018-06-10"), other: "E" }
]
},
{
_id: "5b1522f5cdac0b6da18f761c",
name: 'B',
someArray: [
{ timestamp: new Date("2018-07-04"), other: "D" },
]
}
]);
// Select matching 'documents'
let docs = await Test.find({
"someArray": {
"$elemMatch": {
"timestamp": { "$gte": startDate, "$lt": endDate }
}
}
});
log({ docs });
let filtered = await Test.aggregate([
{ "$match": {
"_id": ObjectId("5b1522f5cdac0b6da18f7618"),
"someArray": {
"$elemMatch": {
"timestamp": { "$gte": startDate, "$lt": endDate }
}
}
}},
{ "$addFields": {
"someArray": {
"$filter": {
"input": "$someArray",
"cond": {
"$and": [
{ "$gte": [ "$$this.timestamp", startDate ] },
{ "$lt": [ "$$this.timestamp", endDate ] }
]
}
}
}
}}
]);
log({ filtered });
mongoose.disconnect();
} catch(e) {
console.error(e)
} finally {
process.exit()
}
})()
Both are the same and give the same output:
Mongoose: filtertest.remove({}, {})
Mongoose: filtertest.insertMany([ { _id: 5b1522f5cdac0b6da18f7618, name: 'A', someArray: [ { _id: 5b1526952794447083ababf6, timestamp: 2018-06-01T00:00:00.000Z, other: 'C' }, { _id: 5b1526952794447083ababf5, timestamp: 2018-07-04T00:00:00.000Z, other: 'D' }, { _id: 5b1526952794447083ababf4, timestamp: 2018-06-10T00:00:00.000Z, other: 'E' } ], __v: 0 }, { _id: 5b1522f5cdac0b6da18f761c, name: 'B', someArray: [ { _id: 5b1526952794447083ababf8, timestamp: 2018-07-04T00:00:00.000Z, other: 'D' } ], __v: 0 } ], {})
Mongoose: filtertest.find({ someArray: { '$elemMatch': { timestamp: { '$gte': new Date("Fri, 01 Jun 2018 00:00:00 GMT"), '$lt': new Date("Sun, 01 Jul 2018 00:00:00 GMT") } } } }, { fields: {} })
{
"docs": [
{
"_id": "5b1522f5cdac0b6da18f7618",
"name": "A",
"someArray": [
{
"_id": "5b1526952794447083ababf6",
"timestamp": "2018-06-01T00:00:00.000Z",
"other": "C"
},
{
"_id": "5b1526952794447083ababf5",
"timestamp": "2018-07-04T00:00:00.000Z",
"other": "D"
},
{
"_id": "5b1526952794447083ababf4",
"timestamp": "2018-06-10T00:00:00.000Z",
"other": "E"
}
],
"__v": 0
}
]
}
Mongoose: filtertest.aggregate([ { '$match': { _id: 5b1522f5cdac0b6da18f7618, someArray: { '$elemMatch': { timestamp: { '$gte': 2018-06-01T00:00:00.000Z, '$lt': 2018-07-01T00:00:00.000Z } } } } }, { '$addFields': { someArray: { '$filter': { input: '$someArray', cond: { '$and': [ { '$gte': [ '$$this.timestamp', 2018-06-01T00:00:00.000Z ] }, { '$lt': [ '$$this.timestamp', 2018-07-01T00:00:00.000Z ] } ] } } } } } ], {})
{
"filtered": [
{
"_id": "5b1522f5cdac0b6da18f7618",
"name": "A",
"someArray": [
{
"_id": "5b1526952794447083ababf6",
"timestamp": "2018-06-01T00:00:00.000Z",
"other": "C"
},
{
"_id": "5b1526952794447083ababf4",
"timestamp": "2018-06-10T00:00:00.000Z",
"other": "E"
}
],
"__v": 0
}
]
}

Resources