nodejs /Mongo - multiple finds not working - node.js

Still a NooB to Node/Mongo and am stuck on this.
I have two mongo collections, Tenants and Rent. Rent collection has the tenant _id in the schema. The following function is searching through all active tenants and for each of those pulling out some attributes for the latest rent document.
The first part of the function populates the tenant object with the results. all working good.
The second .then starts to iterate through the tenant object pulling out the _id to use in the Rent query. (join).
the issue is the for loop seems to iterate through and print the _id correctly, but the second find query seems to only print out the last document in the object. I am just not sure why this is happening
thanks in advance
app.get('/chargerenttest/:date', (req,res) => {
//check date is valid
var rentChargeDate = new Date(req.params.date);
var tenant = "";
//for each active tenant
Tenant .find({
activeTenant : true
})
.then ((tenant) => {
if (!tenant) {
return res.status(404).send();
}
console.log("found tenents")
return tenant
})
.then ((tenant) => {
for (var i in tenant) {
console.log(i)
console.log(tenant[i]._id)
Rent
.find({
"paymentType" :"Rent",
"tenantID" : tenant[i]._id,
"activeEntry": true})
.limit(1)
.sort({datePaid: -1})
// sort in decending date ( latested on top)
.then ((rent) => {
lastPayment = rent[0].datePaid;
lastAmountPaid = rent[0].amountPaid;
console.log("--------",i)
console.log("tenant",tenant[i]._id)
console.log("rentamount",tenant[i].rentAmount)
console.log("lastpayment", lastPayment)
});
}
})
})

Your query can be simplified by running an aggregate operation that makes use of a pipeline with the $lookup operator which allows you to perform a left outer join to another collection in the same database to filter in documents from the "joined" collection for processing.
Consider running the following pipeline:
Rent.aggregate([
{
"$match": {
"paymentType": "Rent",
"activeEntry": true
}
},
{
"$lookup": {
"from": "tenants",
"localField": "tenantID",
"foreignField": "_id",
"as": "tenants"
}
},
{ "$match": { "tenants": { "$ne": [] }, "tenants.activeTenant": true } },
//{ "$unwind": "$tenants" },
{ "$sort": { "datePaid": -1 } },
{ "$limit": 1 }
]).exec((err, rent) => {
if (err) throw err;
lastPayment = rent[0].datePaid;
lastAmountPaid = rent[0].amountPaid;
tenant = rent[0].tenants[0];
console.log("tenant",tenant._id)
console.log("rentamount",tenant.rentAmount)
console.log("lastpayment", lastPayment)
});

This module #coolgk/mongo could make joining multiple collections a lot simpler.
Examples
SQL to Mongo Join
Left Join
SELECT * FROM a LEFT JOIN b ON a.b_id = b.id
becomes
model.find({}, {
join: [ { on: 'b_id' } ]
})
Result:
[{
_id: '5a8bde4ae2ead929f89f3c42',
a_name: 'aname1',
b_id: {
_id: '5a8bde4ae2ead929f89f3c41',
b_name: 'bname1'
}
}, { ... }, ... ]
Inner Join with Constraints
SELECT * FROM a, b WHERE a.b_id = b.id AND b.b_name = 'bname1'
becomes
model.find({}, {
join: [ { on: 'b_id', filters: { b_name: 'bname1' } } ]
})
Result:
[{
_id: '5a8bdfb05d44ea2a08fa8a4c',
a_name: 'aname2',
b_id: {
_id: '5a8bdfb05d44ea2a08fa8a4b',
b_name: 'bname2'
}
}]
Inner Join on Mulitple Collections
SELECT * FROM a, b, c WHERE a.b_id = b.id AND b.c_id = c.id AND c.c_name = 'cname3'
becomes
modela.find({}, {
join: [{
on: 'b_id',
join: [{
on: 'c_id',
filters: { c_name: 'cname3' }
}]
}]
})
Result:
[{
_id: '5a8bdfc1b07af22a12cb1f0b',
a_name: 'aname3',
b_id: {
_id: '5a8bdfc1b07af22a12cb1f0a',
b_name: 'bname3',
c_id: {
_id: '5a8bdfc1b07af22a12cb1f09',
c_name: 'cname3'
}
}
}]

Related

Mongoose: Infinite scroll with filtering

I have these two models:
User.js
const UserSchema = new Schema({
profile: {
type: Schema.Types.ObjectId,
ref: "profiles",
},
following: [
{
type: Schema.Types.ObjectId,
ref: "users",
},
],
});
module.exports = User = mongoose.model("users", UserSchema);
Profile.js
const ProfileSchema = new Schema({
videoURL: {
type: String,
},
});
module.exports = Profile = mongoose.model("profiles", ProfileSchema);
Here's an example of a User document:
{
"following": [
{
"profile":{
"videoURL":"video_url_1"
}
},
{
"profile":{
"videoURL":"video_url_2"
}
},
{
"profile":{}
},
{
"profile":{
"videoURL":"video_url_3"
}
},
{
"profile":{
"videoURL":"video_url_4"
}
},
{
"profile":{
"videoURL":"video_url_5"
}
},
{
"profile":{}
},
{
"profile":{
"videoURL":"video_url_6"
}
}
]
}
I am trying to implement an infinite scroll of the videos of the users followed by the connected user.
This means, I will have to filter user.following.profile.videoURL
WHERE videoURL exists
Suppose, I will be loading two videos, by two videos:
Response 1: ["video_url_1","video_url_2"]
Response 2: ["video_url_3","video_url_4"]
Response 3: ["video_url_5","video_url_6"]
Usually, infinite scroll is easy because all I have to load the documents 2 by 2 by order of storage without filtering on any field.
Example: Displaying the followed users two by two in an infinite scroll
User.findById(user_id).populate({
path: "following",
options: {
skip: 2 * page,
limit: 2,
},
});
But, now I have to perform filtering on each followed_user.profile.video, and return two by two. And I don't see how I can perform BOTH the filtering and the infinite scroll at the same time.
NOTE: According to the documentation:
In general, there is no way to make populate() filter stories based on properties of the story's author. For example, the below query won't return any results, even though author is populated.
const story = await Story.
findOne({ 'author.name': 'Ian Fleming' }).
populate('author').
exec();
story; // null
So I suppose, there is no way for me to use populate to filter based user.followers, based on each user.follower.profile.videoURL
I am not sure it is possible with populate method, but you can try aggregation pipeline,
$match user_id condition
$lookup with aggregation pipeline in users collection for following
$match following id condition
$lookup with profile for following.profile
$match videoURL should exists
$project to show profile field and get first element using $arrayElemAt
$slice to do pagination in following
let page = 0;
let limit = 2;
let skip = limit * page;
User.aggregate([
{ $match: { _id: mongoose.Types.ObjectId(user_id) } },
{
$lookup: {
from: "users",
let: { following: "$following" },
pipeline: [
{ $match: { $expr: { $in: ["$_id", "$$following"] } } },
{
$lookup: {
from: "profiles",
localField: "profile",
foreignField: "_id",
as: "profile"
}
},
{ $match: { "profile.videoURL": { $exists: true } } },
{
$project: {
profile: { $arrayElemAt: ["$profile", 0] }
}
}
],
as: "following"
}
},
{
$addFields: {
following: {
$slice: ["$following", skip, limit]
}
}
}
])
Playground
Suggestion:
You can improve your schema design,
removing profile schema and add profile object in users collection, so you can achieve easily your requirement using populate method,
put match condition in following populate for videoURL exists
const UserSchema = new Schema({
profile: {
type: {
videoURL: {
type: String
}
}
},
following: [
{
type: Schema.Types.ObjectId,
ref: "users"
}
]
});
module.exports = User = mongoose.model("users", UserSchema);
User.findById(user_id).populate({
path: "following",
match: {
"profile.videoURL": { $ne: null }
},
options: {
skip: 2 * page,
limit: 2,
}
});
So what you want is table with infinite scroll and:
You can opt given ways to approach your problem :
Load data (first page) into grid.
Set filter on a col.
Load data again, this time using the filter.

check an array of string value with array of object in mongodb

I have array of strings like this
let fromHour = ['2.5','3','3.5']
let toHour = ['2.5','3','3.5']
I have an array of object saved in mongoDB
timeRange = [
{
from:'2.5',
to:'3'
},
{
from:'3',
to:'3.5'
}
]
I want to check if any of my array of string value exist in that object value
I have tried this but it give me this error ( Unrecognized expression '$match' )
checkAppoint = await Appointment.aggregate([
{
$project: {
date: myScheduleFinal[k].date,
status: { $in: ['pending', 'on-going'] },
timeRange: {
'$match': {
'from': { $in: fromHolder },
'to': { $in: toHolder },
},
},
},
},
]);
also I have tried this solution and it work for me but it take to much time so I am trying this with aggregate
checkAppoint = await Appointment.findOne({
date: myScheduleFinal[k].date,
status: { $in: ['pending', 'on-going'] },
timeRange:{$elemMatch:{
from:{$in:fromHolder},
to:{$in:toHolder}
}}
});
So anyone have a solution for that
Just try $elemMatch and $in operators,
using find() method
checkAppoint = await Appointment.find({
timeRange: {
$elemMatch: {
from: { $in: fromHour },
to: { $in: toHour }
}
}
})
Playground
using aggregate() method
checkAppoint = await Appointment.aggregate([
{
$match: {
timeRange: {
$elemMatch: {
from: { $in: fromHour },
to: { $in: toHour }
}
}
}
}
])
Playground
So I have found a way around to solve this problem and I will share the solution I used
First I want to minimize my request to mongodb so I am now making just one request that bring all the appointment with the required date
and I want to make it this way because my fromHour and toHour array will change many time through single request
helperArray => contains all the day I want to check it's range
let checkAppoint = await Appointment.find({
date: { $in: helperArray },
status: { $in: ['pending', 'on-going'] },
});
now inside my for loop I will go through that data
checkAppoint.filter((singleAppoint) => {
if (singleAppoint._doc.date === myScheduleFinal[k].date) {
singleAppoint._doc.timeRange.map((singleTime) => {
if (fromHolder.includes(singleTime.from)) {
busy = true;
}
});
}
});

Right outer join in aggregation pipeline

I have two collections, let's call them Cats and Parties, with the following schemas:
Cat
{ name: String }
Party
{ date: Date, attendants: [{ cat: { ref: 'Cat' }, role: String }] }
where role symbolizes some other attribute, say, whether the attending cat is a VIP member.
Now I want to get a list of all cats that exist (even those poor kitties who never attended any party) and for each cat, I want a list of all the roles it ever had for at least one party. Furthermore, I want this entire list to be sorted by the (per cat) last attended party's date with cats who never attended any party being last.
This raises the following problems for me:
Aggregrating over Parties excludes party-pooper kitties who never joined a party.
Aggregating over Cats sort of goes »the wrong way« because I cannot $lookup parties the cat attended because that information is in a subdocument array.
The pipeline I currently have gives me all cats who attended at least one party with a list of their roles, but doesn't sort by the last attended party. In fact, I could live with excluding cats who never attended a party, but the sorting is crucial for me:
Party.aggregate([
{ $unwind: '$attendants' },
{ $project: { role: '$attendants.role', cat: '$attendants.cat' } },
{
$group: {
_id: '$cat',
roles: { $addToSet: '$role' }
}
},
{
$lookup: {
from: 'cats',
localField: '_id',
foreignField: '_id',
as: 'cat'
}
},
{ $unwind: '$cat' },
// (*)
{ $addFields: { 'cat.roles': '$roles' } },
{ $replaceRoot: { newRoot: '$cat' } }
])
My current idea would basically be a right outer join at (*) to add a list of parties the cat has attended, $project that to the party's date and then $group using $max to get the latest date. Then I can $unwind that now one-element array and $sort over it in the end.
The problem is that right outer joins don't exist in mongo, AFAIK, and I don't know how to get that list of parties per cat within the pipeline.
To clarify, the expected output should be something like
[
{
"_id": "59982d3c7ca25936f8c327c8",
"name": "Mr. Kitty",
"roles": ["vip", "birthday cat"],
"dateOfLastParty": "2017-06-02"
},
{
"_id": "59982d3c7ca25936f8c327c9",
"name": "Snuffles",
"roles": ["best looking cat"],
"dateOfLastParty": "2017-06-01"
},
...
{
"_id": "59982d3c7ca25936f8c327c4",
"name": "Sad Face McLazytown",
"roles": [],
"dateOfLastParty": null
},
]
As stated, you want the "cats" so use the Cat model and do the "left outer join" that is actually inherent to $lookup, rather than asking for a "right outer join" from the opposing collection, since a "right outer join" is not possible with MongoDB at this time.
It's also far more practical as a "left join", because you want "cats" as your primary source of output. The only thing to consider when linking to "Party" is that each "Cat" is listed in an array, and therefore you get the whole document back. So all that needs to be done is in "post processing" after the $lookup, you simply "filter" the array content for the matching entry of the current cat.
Fortunately we get good features with $arrayElemAt and $indexOfArray, that allow us to do that exact extraction:
let kitties = await Cat.aggregate([
{ '$lookup': {
'from': Party.collection.name,
'localField': '_id',
'foreignField': 'attendants.cat',
'as': 'parties'
}},
{ '$replaceRoot': {
'newRoot': {
'$let': {
'vars': {
'parties': {
'$map': {
'input': '$parties',
'as': 'p',
'in': {
'date': '$$p.date',
'role': {
'$arrayElemAt': [
'$$p.attendants.role',
{ '$indexOfArray': [ '$$p.attendants.cat', '$_id' ] }
]
}
}
}
}
},
'in': {
'_id': '$_id',
'name': '$name',
'roles': '$$parties.role',
'dateOfLastParty': { '$max': '$$parties.date' }
}
}
}
}}
]);
So my concept of "optimal" processing here actually uses $replaceRoot here because you can define the whole document under a $let statement. The reason I'm doing that is so we can take the "parties" array output from the previous $lookup and reshape each entry extracting the matching "role" data for the current "kitty" at that given party. This we can actually make a variable itself.
The reason for the "array variable" is because we can then use $max to extract the "largest/last" date property as "singular" and still extract the "role" values as an "array" from that reshaped content. This makes it easy to define the fields you wanted.
And since it was a "left join" started from Cat in the first place, then those poor kitties that missed out on all parties are still there, and still have the desired output.
Two aggregation pipeline stages. What could be more simple!
As a full listing:
const mongoose = require('mongoose'),
Schema = mongoose.Schema;
mongoose.Promise = global.Promise;
mongoose.set('debug',true);
const uri = 'mongodb://localhost/catparty',
options = { useMongoClient: true };
const catSchema = new Schema({
name: String
});
const partySchema = new Schema({
date: Date,
attendants: [{
cat: { type: Schema.Types.ObjectId, ref: 'Cat' },
role: String
}]
});
const Cat = mongoose.model('Cat', catSchema);
const Party = mongoose.model('Party', partySchema);
function log(data) {
console.log(JSON.stringify(data,undefined,2))
}
(async function() {
try {
const conn = await mongoose.connect(uri,options);
// Clean collections
await Promise.all(
Object.keys(conn.models).map( m => conn.models[m].remove({}) )
);
var cats = await Cat.insertMany(
['Fluffy', 'Snuggles', 'Whiskers', 'Socks'].map( name => ({ name }) )
);
cats.shift();
cats = cats.map( (cat,idx) =>
({ cat: cat._id, role: (idx === 0) ? 'Host' : 'Guest' })
);
log(cats);
let party = await Party.create({
date: new Date(),
attendants: cats
});
log(party);
let kitties = await Cat.aggregate([
{ '$lookup': {
'from': Party.collection.name,
'localField': '_id',
'foreignField': 'attendants.cat',
'as': 'parties'
}},
{ '$replaceRoot': {
'newRoot': {
'$let': {
'vars': {
'parties': {
'$map': {
'input': '$parties',
'as': 'p',
'in': {
'date': '$$p.date',
'role': {
'$arrayElemAt': [
'$$p.attendants.role',
{ '$indexOfArray': [ '$$p.attendants.cat', '$_id' ] }
]
}
}
}
}
},
'in': {
'_id': '$_id',
'name': '$name',
'roles': '$$parties.role',
'dateOfLastParty': { '$max': '$$parties.date' }
}
}
}
}}
]);
log(kitties);
} catch(e) {
console.error(e);
} finally {
mongoose.disconnect();
}
})();
And example output:
[
{
"_id": "59a00d9528683e0f59e53460",
"name": "Fluffy",
"roles": [],
"dateOfLastParty": null
},
{
"_id": "59a00d9528683e0f59e53461",
"name": "Snuggles",
"roles": [
"Host"
],
"dateOfLastParty": "2017-08-25T11:44:21.903Z"
},
{
"_id": "59a00d9528683e0f59e53462",
"name": "Whiskers",
"roles": [
"Guest"
],
"dateOfLastParty": "2017-08-25T11:44:21.903Z"
},
{
"_id": "59a00d9528683e0f59e53463",
"name": "Socks",
"roles": [
"Guest"
],
"dateOfLastParty": "2017-08-25T11:44:21.903Z"
}
]
And you should be able to see how those "roles" values actually become an array with more data. And if you need that to be a "unique list", then simply wrap with $setDifference as in:
'roles': { '$setDifference': [ '$$parties.role', [] ] },
And that is also covered

Mongoose sort the aggregated result

I'm having a lot of difficulty in solving this mongodb (mongoose) problem.
There is this schema 'Recommend' (username, roomId, ll and date) and its collection contains recommendation of user.
I need to get a list of most recommended rooms (by roomId). Below is the schema and my tried solution with mongoose query.
var recommendSchema = mongoose.Schema({
username: String,
roomId: String,
ll: { type: { type: String }, coordinates: [ ] },
date: Date
})
recommendSchema.index({ ll: '2dsphere' });
var Recommend = mongoose.model('Recommend', recommendSchema);
Recommend.aggregate(
{
$group:
{
_id: '$roomId',
recommendCount: { $sum: 1 }
}
},
function (err, res) {
if (err) return handleError(err);
var resultSet = res.sort({'recommendCount': 'desc'});
}
);
The results returned from the aggregation pipeline are just plain objects. So you do the sorting as a pipeline stage, not as a separate operation:
Recommend.aggregate(
[
// Grouping pipeline
{ "$group": {
"_id": '$roomId',
"recommendCount": { "$sum": 1 }
}},
// Sorting pipeline
{ "$sort": { "recommendCount": -1 } },
// Optionally limit results
{ "$limit": 5 }
],
function(err,result) {
// Result is an array of documents
}
);
So there are various pipeline operators that can be used to $group or $sort or $limit and other things as well. These can be presented in any order, and as many times as required. Just understanding that one "pipeline" stage flows results into the next to act on.

Get result as an array instead of documents in mongodb for an attribute

I have a User collection with schema
{
name: String,
books: [
id: { type: Schema.Types.ObjectId, ref: 'Book' } ,
name: String
]
}
Is it possible to get an array of book ids instead of object?
something like:
["53eb797a63ff0e8229b4aca1", "53eb797a63ff0e8229b4aca2", "53eb797a63ff0e8229b4aca3"]
Or
{ids: ["53eb797a63ff0e8229b4aca1", "53eb797a63ff0e8229b4aca2", "53eb797a63ff0e8229b4aca3"]}
and not
{
_id: ObjectId("53eb79d863ff0e8229b97448"),
books:[
{"id" : ObjectId("53eb797a63ff0e8229b4aca1") },
{ "id" : ObjectId("53eb797a63ff0e8229b4acac") },
{ "id" : ObjectId("53eb797a63ff0e8229b4acad") }
]
}
Currently I am doing
User.findOne({}, {"books.id":1} ,function(err, result){
var bookIds = [];
result.books.forEach(function(book){
bookIds.push(book.id);
});
});
Is there any better way?
It could be easily done with Aggregation Pipeline, using $unwind and $group.
db.users.aggregate({
$unwind: '$books'
}, {
$group: {
_id: 'books',
ids: { $addToSet: '$books.id' }
}
})
the same operation using mongoose Model.aggregate() method:
User.aggregate().unwind('$books').group(
_id: 'books',
ids: { $addToSet: '$books.id' }
}).exec(function(err, res) {
// use res[0].ids
})
Note that books here is not a mongoose document, but a plain js object.
You can also add $match to select some part of users collection to run this aggregation query on.
For example, you may select only one particular user:
User.aggregate().match({
_id: uid
}).unwind('$books').group(
_id: 'books',
ids: { $addToSet: '$books.id' }
}).exec(function(err, res) {
// use res[0].ids
})
But if you're not interested in aggregating books from different users into single array, it's best to do it without using $group and $unwind:
User.aggregate().match({
_id: uid
}).project({
_id: 0,
ids: '$books.id'
}).exec(function(err, users) {
// use users[0].ids
})

Resources