MongoDB greaterThan many values - node.js

I have an API built with Node + Mongoose, and now I want to filter the data using some fields. Everything works fine, but how can I get the data like: 1 < age < 3 AND 8 < age < 11
I am trying this:
query.where('age').gte(1).lte(3);
query.where('age').gte(8).lte(11);
but this code only gets me 8 < age < 11
Thanks!

You mean $or, since an $and condition cannot possibly overlap where you condition would be true:
{
"$or": [
{ "age": { "$gte": 1, "$lte": 3 } },
{ "age": { "$gte": 8, "$lte": 11 } }
]
}
Also you are using JavaScript which has a nice free flowing object notation, so the helper methods are really overkill here. Use the standard query operator syntax instead.
Just a demo to show different forms of representing the same query:
var mongoose = require('mongoose'),
Schema = mongoose.Schema;
var testSchema = new Schema({
"age": Number
});
var Test = mongoose.model( 'Test', testSchema, 'testcol' );
var query = Test.where({
"$or": [
{ "age": { "$gte": 1, "$lte": 3 } },
{ "age": { "$gte": 8, "$lte": 11 } }
]
});
console.log( JSON.stringify( query._conditions, undefined, 2 ) );
var query2 = Test.where().or([
{ "age": { "$gte": 1, "$lte": 3 } },
{ "age": { "$gte": 8, "$lte": 11 } }
]);
console.log( JSON.stringify( query2._conditions, undefined, 2 ) );
var query3 = Test.where().or(
[
Test.where("age").gte(1).lte(3)._conditions,
Test.where("age").gte(8).lte(11)._conditions
]
);
console.log( JSON.stringify( query3._conditions, undefined, 2 ) );
Which should also demonstrate that the "helpers" are not really adding much value to how the query is basically formed.

Related

Mongoose Filter based on Dynamic Date Key with value

I have created an employee attendance application where attendances are logged and stored in a database. I have tried to obtain a count of all date-field with the value of "Present". Data are stored in the database like so :
"attendances": { <YYYY-MM-DD>: "value" } pair
// The values being "Absent" or "Present" whatever the case may be.
The problem is, I get a value of 0 whenever I try to count all the entries with "attendances": {"2019-08-28": "Present"}.
Can anyone help me find out what am doing wrong?
//Schema
const Schema = mongoose.Schema;
const employeeSchema = new Schema({
name: String,
department: String,
origin: String,
employDate: String,
attendances: Object
});
module.exports= Employee = mongoose.model('Employee', employeeSchema);
route.js
router.get('/',(req,res)=>{
Employee.collection.countDocuments({"attendances.date":"present"},(err,data)=>{
if(err){
res.status(500)
res.send(err)
}else{
res.status(200)
res.json(data)
}
})
})
//Data stored in MongoDB
{
"_id": "5d6565236574b1162c349d8f",
"name": "Benjamin Hall",
"department": "IT",
"origin": "Texas",
"employDate": "2019-08-27",
"__v": 0,
"attendances": {
"2019-08-28": "Sick"
}
},
{
"_id": "5d6367ee5b78d30c74be90e6",
"name": "Joshua Jaccob",
"department": "Marketing",
"origin": "new york",
"employDate": "2019-08-26",
"__v": 0,
"attendances": {
"2019-08-26": "Present",
"2019-08-27": "Sick"
}
},
If you want to find by property in embedded document you have to use dot notation
this will not work, because you are asking mongoo to find the document which have attendances object equal the same given object.
{ "attendances": {"2019-08-26": "Present"}}
this will work only if attendances object in your database contains only
{ "attendances": {"2019-08-26": "Present"}}
that's mean that you asking mongoo if the stored object is equal the given object and it will return false
{ "attendances": {"2019-08-26": "Present" , "2019-08-27": "Sick"}} == { "attendances": {"2019-08-26": "Present"}}
to do this you have to use dot notation
Employee.collection.countDocuments({"attendances.2019-08-26":"Present"},(err,data)=>{
if(err){
res.status(500)
res.send(err)
}else{
res.status(200)
res.json(data)
}
})
Since the dynamic date is part of an embedded document, to query on that field with a regex expression (for case insensitive search) you essentially need to use the dot notation { "attendance.2019-08-28": /present/i }, constructed using computed property names as:
const date = "2019-08-28" // dynamic date
const query = {
["attendances." + date]: /present/i // computed property name
}
Employee.countDocuments(query, (err, data) => {
if (err){
res.status(500).send(err)
} else{
res.status(200).json(data)
}
})
Note, countDocuments() function can be accessed directly on the Mongoose model.
For a date range query, say for example you want to return the count of attendances that were present for the last 30 days, you would need
to query with the aggregation framework which exposes operators like $objectToArray, $filter and $size to give you the count.
The above operators allow you to convert the attendances document into an array of key value pairs with $objectToArray which you can then filter based on the past 30 days criteria as well as the "present" value using $filter. To get the count, use the $size operator on the filtered array.
As an illustration, applying $objectToArray on the document
{
"2019-08-26": "Present",
"2019-08-27": "Sick"
}
returns
[
{ "k": "2019-08-26", "v": "Present" },
{ "k": "2019-08-27", "v": "Sick" }
]
To filter on the past n days you will need to first create a list of dates in that range i.e.
[
"2019-08-27",
"2019-08-26",
"2019-08-25",
...
]
which can be done in JavaScript as
function formatDate(date) {
var d = new Date(date),
month = '' + (d.getMonth() + 1),
day = '' + d.getDate(),
year = d.getFullYear();
if (month.length < 2) month = '0' + month;
if (day.length < 2) day = '0' + day;
return [year, month, day].join('-');
}
const listDatesForThePastDays = n => (
Array(n)
.fill(new Date())
.map((today, i) => today - 8.64e7 * i)
.map(formatDate)
)
This list can be used in the $filter as
{ "$filter": {
"input": { "$objectToArray": "$attendances" },
"cond": {
"$and": [
{ "$in": ["$$this.k", listDatesForThePastDays(30)] },
{ "$eq": ["$$this.v", "Present"] }
]
}
} }
And apply the $size operator to get the count:
{ "$size": {
"$filter": {
"input": { "$objectToArray": "$attendances" },
"cond": {
"$and": [
{ "$in": ["$$this.k", listDatesForThePastDays(30)] },
{ "$eq": ["$$this.v", "Present"] }
]
}
}
} }
Your overall query will look like
function formatDate(date) {
var d = new Date(date),
month = '' + (d.getMonth() + 1),
day = '' + d.getDate(),
year = d.getFullYear();
if (month.length < 2) month = '0' + month;
if (day.length < 2) day = '0' + day;
return [year, month, day].join('-');
}
const listDatesForThePastDays = n => (
Array(n)
.fill(new Date())
.map((today, i) => today - 8.64e7 * i)
.map(formatDate)
)
Employee.aggregate([
{ "$addFields": {
"numberofPresentAttendances": {
"$size": {
"$filter": {
"input": { "$objectToArray": "$attendances" },
"cond": {
"$and": [
{ "$in": ["$$this.k", listDatesForThePastDays(30)] },
{ "$eq": ["$$this.v", "Present"] }
]
}
}
}
}
} }
]).exec().
.then(results => {
console.log(results);
// results will be an array of employee documents with an extra field numberofPresentAttendances
})
.catch(console.error)
To get the count for all employees then you need to group all the documents as
Employee.aggregate([
{ "$group": {
"_id": null,
"totalPresent": {
"$sum": {
"$size": {
"$filter": {
"input": { "$objectToArray": "$attendances" },
"cond": {
"$and": [
{ "$in": ["$$this.k", listDatesForThePastDays(30)] },
{ "$eq": ["$$this.v", "Present"] }
]
}
}
}
}
}
} }
]).exec()
.then(results => {
console.log(results);
// results will be an array of employee documents with an extra field numberofPresentAttendances
})
.catch(console.error)

MongoDB query and projection on subdocument array returns also array of another document

I am trying to query an embedded subdocument and then only return an array in that subdocument via projection. After a query you can select fields that you want returned via projection. I want to use the native functionality because it is possible and the most clean way. The problem is it returns arrays in two documents.
I tried different query and projection options, but no result.
User model
// Define station schema
const stationSchema = new mongoose.Schema({
mac: String,
stationName: String,
syncReadings: Boolean,
temperature: Array,
humidity: Array,
measures: [{
date: Date,
temperature: Number,
humidity: Number
}],
lastUpdated: Date
});
// Define user schema
var userSchema = mongoose.Schema({
apiKey: String,
stations : [stationSchema]
}, {
usePushEach: true
}
);
api call
app.get('/api/stations/:stationName/measures',function(req, res, next) {
var user = {
apiKey: req.user.apiKey
}
const query = {
apiKey: user.apiKey,
'stations.stationName': req.params.stationName
}
const options = {
'stations.$.measures': 1,
}
User.findOne(query, options)
.exec()
.then(stations => {
res.status(200).send(stations)
})
.catch(err => {
console.log(err);
res.status(400).send(err);
})
});
Expected result
{
"_id": "5c39c99356bbf002fb092ce9",
"stations": [
{
"stationName": "livingroom",
"measures": [
{
"humidity": 60,
"temperature": 20,
"date": "2019-01-12T22:49:45.468Z",
"_id": "5c3a6f09fd357611f8d078a0"
},
{
"humidity": 60,
"temperature": 20,
"date": "2019-01-12T22:49:46.500Z",
"_id": "5c3a6f0afd357611f8d078a1"
},
{
"humidity": 60,
"temperature": 20,
"date": "2019-01-12T22:49:47.041Z",
"_id": "5c3a6f0bfd357611f8d078a2"
}
]
}
]
}
Actual result
{
"_id": "5c39c99356bbf002fb092ce9",
"stations": [
{
"stationName": "livingroom",
"measures": [
{
"humidity": 60,
"temperature": 20,
"date": "2019-01-12T22:49:45.468Z",
"_id": "5c3a6f09fd357611f8d078a0"
},
{
"humidity": 60,
"temperature": 20,
"date": "2019-01-12T22:49:46.500Z",
"_id": "5c3a6f0afd357611f8d078a1"
},
{
"humidity": 60,
"temperature": 20,
"date": "2019-01-12T22:49:47.041Z",
"_id": "5c3a6f0bfd357611f8d078a2"
}
]
},
******************************************************
// this whole object should not be returned
{
"stationName": "office",
"measures": []
}
******************************************************
]
}
edit
The answer below with aggregation works, but I still find it odd that I would need so much code. If after my normal query I get the same result with ".stations[0].measures", instead of the whole aggregation pipeline:
.then(stations => {
res.status(200).send(stations.stations[0].measures)
})
The way I read the code, the above does exactly the same as:
const options = {'stations.$.measures': 1}
Where the dollar sign puts in the index 0 as that was the index of the station that matches the query part: stationName: "livingroom"
Can someone explain?
This is not described in terms of mongoose but this will find a particular station name in an array of stations in 1 or more docs and return only the measures array:
db.foo.aggregate([
// First, find the docs we are looking for:
{$match: {"stations.stationName": "livingroom"}}
// Got the doc; now need to fish out ONLY the desired station. The filter will
// will return an array so use arrayElemAt 0 to extract the object at offset 0.
// Call this intermediate qqq:
,{$project: { qqq:
{$arrayElemAt: [
{ $filter: {
input: "$stations",
as: "z",
cond: { $eq: [ "$$z.stationName", "livingroom" ] }
}}, 0]
}
}}
// Lastly, just project measures and not _id from this object:
,{$project: { _id:0, measures: "$qqq.measures" }}
]);
$elemMatch operator limits the contents of an array field from the query results to contain only the first element matching the $elemMatch condition.
Try $elemMatch in Select Query as below :
const query = {
apiKey: user.apiKey,
'stations.stationName': req.params.stationName
}
const options = {
'stations' : {$elemMatch: { 'stationName' : req.params.stationName }}
}

Set Incremental Values from Array of Items

How to update the multiple documents in MongoDB and set the value of the element in an increasing order?
I have got the document as follows
{
"_id" : ObjectId("5b162a31dfaf342dc44c920d")
}
{
"_id" : ObjectId("5b162a31dfaf342dc44c920f")
}
{
"_id" : ObjectId("5b162a31dfaf342dc44c920c")
}
How can I update the whole documents with a single query so that I can have a new element called "order" in every single field in an increasing order as below
{
"_id" : ObjectId("5b162a31dfaf342dc44c920d"),
"order": 1
}
{
"_id" : ObjectId("5b162a31dfaf342dc44c920f"),
"order": 2
}
{
"_id" : ObjectId("5b162a31dfaf342dc44c920c"),
"order": 3
}
Currently I am using the following way to solve the problem
for(let i = 0; i <= req.body.id.length;i++) {
const queryOpts = {
_id: ObjectId(req.body.id[i])
};
const updateOpts = {
$set: {
'order': i + 1
}
};
const dataRes = await req.db.collection('GalleryImage').updateOne(queryOpts, updateOpts);
if(i === req.body.id.length-1) {
return commonHelper.sendResponseMessage(res, dataRes, {
_id: req.body.id
}, moduleConfig.message.updateGalleryOrder);
}
If there any better way than this so that it would not be the expensive operation if there are large number of documents ?
Use bulkWrite() with Array.map() to construct the statement:
try {
let response = await req.db.collection('GalleryImage').bulkWrite(
req.body.id.map((_id,order) =>
({ updateOne: {
filter: { _id: ObjectId(_id) },
update: {
$set: { order: order+1 }
}
}})
)
);
} catch(e) {
// deal with any errors
}
Array.map() has the "index" of the array element being processed within it's second function argument. So simply use that to get the order and set that on all statements.
Rather than writing/responding with the database n times, this only needs happen "once".
There is no other way to get a "sequence" other than introducing it yourself, but at least we can do it with "one" write this way instead of several. Note also to "trap your possible errors" when using async/await syntax.
Example listing
const { MongoClient, ObjectID: ObjectId } = require('mongodb');
const uri = 'mongodb://localhost:27017';
const data = [
"5b162a31dfaf342dc44c920d",
"5b162a31dfaf342dc44c920f",
"5b162a31dfaf342dc44c920c"
];
const log = data => console.log(JSON.stringify(data, undefined, 2));
(async function() {
try {
const client = await MongoClient.connect(uri);
let db = client.db('test');
// Set up
await db.collection('gallery').removeMany({});
await db.collection('gallery').insertMany(
data.map(_id => ({ _id: ObjectId(_id) }))
);
// Update with indexes
let response = await db.collection('gallery').bulkWrite(
data.map((_id,idx) =>
({
updateOne: {
filter: { _id: ObjectId(_id) },
update: { $set: { order: idx+1 } }
}
})
)
);
log({ response });
let items = await db.collection('gallery').find().toArray();
log({ items });
client.close();
} catch(e) {
console.error(e)
} finally {
process.exit()
}
})()
And the output
{
"response": {
"ok": 1,
"writeErrors": [],
"writeConcernErrors": [],
"insertedIds": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 3,
"nModified": 3,
"nRemoved": 0,
"upserted": [],
"lastOp": {
"ts": "6563535160225038345",
"t": 18
}
}
}
{
"items": [
{
"_id": "5b162a31dfaf342dc44c920d",
"order": 1
},
{
"_id": "5b162a31dfaf342dc44c920f",
"order": 2
},
{
"_id": "5b162a31dfaf342dc44c920c",
"order": 3
}
]
}
Clearly shows nMatched: 3 and nModified: 3 just as is expected.

MongoDB aggregation group average rating by last 7 days with empty values

I'm attempting to query a collection and retrieve an average value for the each of the last 7 days excluding the current day. On some or all of the days there may not be an average.
Here's what I have so far:
var dateTill = moment({hour:0,minute:0}).subtract(1, 'days')._d
var dateSevenDaysAgo = moment({hour:0,minute:0}).subtract(7, 'days')._d;
Rating.aggregate([
{
$match:{
userTo:facebookId,
timestamp:{$gt:dateSevenDaysAgo,$lt:dateTill}
}
},
{
$group:{
_id:{day:{'$dayOfMonth':'$timestamp'}},
average:{$avg:'$rating'}
}
},
{
$sort:{
'_id.day':1
}
}
]
This gives me
[ { _id: { day: 20 }, average: 1 },
{ _id: { day: 22 }, average: 3 },
{ _id: { day: 24 }, average: 5 } ]
What I'm trying to get is something like:
[1,,3,,5,,]
Which represents the last 7 days of averages in order and has an empty element where there is no average for that day.
I could try and make a function that detects where the gaps are but this won't work when the averages are spread across two different months. e.g (July 28,29,30,31,Aug 1,2] - the days in august will be sorted to the front of the array I want.
Is there an easier way to do this?
Thanks!
People ask about "empty results" quite often, and the thinking usually comes from how they would have approached the problem with a SQL query.
But whilst it is "possible" to throw a set of "empty results" for items that do not contain a grouping key, it is a difficult process and much like the SQL approach people use, it's just throwing those values within the statement artificially and it really isn't a very performance driven alternative. Think "join" with a manufactured set of keys. Not efficient.
The smarter approach is to have those results ready in the client API directly, without sending to the server. Then the aggregation output can be "merged" with those results to create a complete set.
However you want to store the set to merge with is up to you, it just requires a basic "hash table" and lookups. But here is an example using nedb, which allows you to maintain the MongoDB set of thinking for query and updates:
var async = require('async'),
mongoose = require('mongoose'),
DataStore = require('nedb'),
Schema = mongoose.Schema,
db = new DataStore();
mongoose.connect('mongodb://localhost/test');
var Test = mongoose.model(
'Test',
new Schema({},{ strict: false }),
"testdata"
);
var testdata = [
{ "createDate": new Date("2015-07-20"), "value": 2 },
{ "createDate": new Date("2015-07-20"), "value": 4 },
{ "createDate": new Date("2015-07-22"), "value": 4 },
{ "createDate": new Date("2015-07-22"), "value": 6 },
{ "createDate": new Date("2015-07-24"), "value": 6 },
{ "createDate": new Date("2015-07-24"), "value": 8 }
];
var startDate = new Date("2015-07-20"),
endDate = new Date("2015-07-27"),
oneDay = 1000 * 60 * 60 * 24;
async.series(
[
function(callback) {
Test.remove({},callback);
},
function(callback) {
async.each(testdata,function(data,callback) {
Test.create(data,callback);
},callback);
},
function(callback) {
async.parallel(
[
function(callback) {
var tempDate = new Date( startDate.valueOf() );
async.whilst(
function() {
return tempDate.valueOf() <= endDate.valueOf();
},
function(callback) {
var day = tempDate.getUTCDate();
db.update(
{ "day": day },
{ "$inc": { "average": 0 } },
{ "upsert": true },
function(err) {
tempDate = new Date(
tempDate.valueOf() + oneDay
);
callback(err);
}
);
},
callback
);
},
function(callback) {
Test.aggregate(
[
{ "$match": {
"createDate": {
"$gte": startDate,
"$lt": new Date( endDate.valueOf() + oneDay )
}
}},
{ "$group": {
"_id": { "$dayOfMonth": "$createDate" },
"average": { "$avg": "$value" }
}}
],
function(err,results) {
if (err) callback(err);
async.each(results,function(result,callback) {
db.update(
{ "day": result._id },
{ "$inc": { "average": result.average } },
{ "upsert": true },
callback
)
},callback);
}
);
}
],
callback
);
}
],
function(err) {
if (err) throw err;
db.find({},{ "_id": 0 }).sort({ "day": 1 }).exec(function(err,result) {
console.log(result);
mongoose.disconnect();
});
}
);
Which gives this output:
[ { day: 20, average: 3 },
{ day: 21, average: 0 },
{ day: 22, average: 5 },
{ day: 23, average: 0 },
{ day: 24, average: 7 },
{ day: 25, average: 0 },
{ day: 26, average: 0 },
{ day: 27, average: 0 } ]
In short, a "datastore" is created with nedb, which basically acts the same as any MongoDB collection ( with stripped down functionality ). You then insert your range of "keys" expected and default values for any of the results.
Then running your aggregation statement, which is only going to return the keys that exist in the queried collection, you simply "update" the created datastore at the same key with the aggregated values.
To make that a bit more efficient, I am running both the empty result "creation" and the "aggregation" operations in parallel, utilizing "upsert" functionallity and the $inc operator for the values. These will not conflict, and that means the creation can happen at the same time as the aggregation is running, so no delays.
This is very simple to integrate into your API, so you can have all the keys you want, including those with no data for aggregation in the collection for output.
The same approach adapts well to using another actual collection on your MongoDB server for very large result sets. But if they are very large, then you should be pre-aggregating results anyway, and just using standard queries to sample.

Sequelize OR condition object

By creating object like this
var condition=
{
where:
{
LastName:"Doe",
FirstName:["John","Jane"],
Age:{
gt:18
}
}
}
and pass it in
Student.findAll(condition)
.success(function(students){
})
It could beautifully generate SQL like this
"SELECT * FROM Student WHERE LastName='Doe' AND FirstName in ("John","Jane") AND Age>18"
However, It is all 'AND' condition, how could I generate 'OR' condition by creating a condition object?
Seems there is another format now
where: {
LastName: "Doe",
$or: [
{
FirstName:
{
$eq: "John"
}
},
{
FirstName:
{
$eq: "Jane"
}
},
{
Age:
{
$gt: 18
}
}
]
}
Will generate
WHERE LastName='Doe' AND (FirstName = 'John' OR FirstName = 'Jane' OR Age > 18)
See the doc: http://docs.sequelizejs.com/en/latest/docs/querying/#where
String based operators will be deprecated in the future (You've probably seen the warning in console).
Getting this to work with symbolic operators was quite confusing for me, and I've updated the docs with two examples.
Post.findAll({
where: {
[Op.or]: [{authorId: 12}, {authorId: 13}]
}
});
// SELECT * FROM post WHERE authorId = 12 OR authorId = 13;
Post.findAll({
where: {
authorId: {
[Op.or]: [12, 13]
}
}
});
// SELECT * FROM post WHERE authorId = 12 OR authorId = 13;
Use Sequelize.or:
var condition = {
where: Sequelize.and(
{ name: 'a project' },
Sequelize.or(
{ id: [1,2,3] },
{ id: { lt: 10 } }
)
)
};
Reference (search for Sequelize.or)
Edit: Also, this has been modified and for the latest method see Morio's answer,
In Sequelize version 5 you might also can use this way (full use Operator Sequelize) :
var condition =
{
[Op.or]: [
{
LastName: {
[Op.eq]: "Doe"
},
},
{
FirstName: {
[Op.or]: ["John", "Jane"]
}
},
{
Age:{
[Op.gt]: 18
}
}
]
}
And then, you must include this :
const Op = require('Sequelize').Op
and pass it in :
Student.findAll(condition)
.success(function(students){
//
})
It could beautifully generate SQL like this :
"SELECT * FROM Student WHERE LastName='Doe' OR FirstName in ("John","Jane") OR Age>18"
For Sequelize 4
Query
SELECT * FROM Student WHERE LastName='Doe'
AND (FirstName = "John" or FirstName = "Jane") AND Age BETWEEN 18 AND 24
Syntax with Operators
const Op = require('Sequelize').Op;
var r = await to (Student.findAll(
{
where: {
LastName: "Doe",
FirstName: {
[Op.or]: ["John", "Jane"]
},
Age: {
// [Op.gt]: 18
[Op.between]: [18, 24]
}
}
}
));
Notes
Avoid alias operators $ (e.g $and, $or ...) as these will be deprecated
Unless you have {freezeTableName: true} set in the table model then Sequelize will query against the plural form of its name ( Student -> Students )
See the docs about querying.
It would be:
$or: [{a: 5}, {a: 6}] // (a = 5 OR a = 6)
where: {
[Op.or]: [
{
id: {
[Op.in]: recordId,
},
}, {
id: {
[Op.eq]: recordId,
},
},
],
},
This Works For Me !
For those who are facing issue in making more complex query like -
// where email = 'xyz#mail.com' AND (( firstname = 'first' OR lastname = 'last' ) AND age > 18)
would be:
[Op.and]: [
{
"email": { [Op.eq]: 'xyz#mail.com' }
// OR "email": 'xyz#mail.com'
},
{
[Op.and]: [
{
[Op.or]: [
{
"firstname": "first"
},
{
"lastname": "last"
}
]
},
{
"age": { [Op.gt]: 18 }
}]
}
]

Resources