How can I do pagination with mongoose? - node.js

All of the other solutions use skip() which is apparently bad for speed. Also I have seen people use nin which seems like it could be a bad solution if you have a bunch of ids that you are checking. Here is what I have so far
//Get the top 20 posts initially
exports.getPosts = (req, res) => {
db.post.find().sort({
numberOfLikes: -1
}).limit(20).exec(function(err, posts){
if(err){
return res.status(500).send();
}
res.json(posts);
});
};
//This is to enable pagination on posts
exports.getMorePosts = (req, res) => {
var query = req.query;
if(!query.hasOwnProperty('numberOfLikes') || !query.hasOwnProperty('postId')){
return res.status(400).send();
}
db.post.find({
numberOfLikes: {
$lte: query.numberOfLikes
},
_id: {
$ne: query.postId
}
}).sort('-numberOfLikes').limit(20).exec(function(err, posts){
if(err){
return res.status(500).send();
}
res.json(posts);
});
};
So I use the first function to get the first 20 posts and then the second to get 20 more posts and so on. The second function get passed the smallest number of likes from the original 20 and the last post's postId from the original 20. From there it gets 20 more.
I want to sort them by the number of likes but the problem is once it gets posts with 0 likes, it never knows when to end and it just keeps cycling back and fourth.

Related

How can I retrieve documents' properties from a pre hook?

I posted this question yesterday because I didn't know how to solve my problem.
Change variable value in document after some time passes?
I was told I need to use a pre hook. I tried to do it, but "this" would refer to the query, not to the document. So I couldn't retrieve the documents to check if the 4 weeks passed. (check the question, you will get it)
Because I don't know how to make this .pre('find') to use variables from each of my document (so it checks if the 4 weeks passed) I was thinking about looping through all of them and checking if 4 weeks passed.
router.get('/judet/:id([0-9]{2})', middleware.access2, function(req, res)
{
var title = "Dashboard";
Somer.find({}, function(err, someri)
{
if(err)
{
console.log(err);
}
else
{
res.render("dashboard", {title: title, id:req.params.id, someri:someri});
}
});
}); ///get route
var someriSchema = new mongoose.Schema({
nume: {type: String, required: true},
dateOfIntroduction: {type:Date, default: Date.now, get: formatareData},
});
someriSchema.pre('find', function(next) {
console.log(this.dateOfIntroduction); <- this will return undefined, because this refers to the query, actually
next();
});///schema and the pre hook. I thought I could use it like this, and inside the body of the pre hook I can check for the date
Here's what I am talking about:
router.get('/judet/:id([0-9]{2})', middleware.access2, function(req, res)
{
var title = "Dashboard | Best DAVNIC73";
Somer.find({}, function(err, someri)
{
if(err)
{
console.log(err);
}
else
{
someri.forEach(function(somer)
{
///check if 4 weeks passed and then update the deactivate variable
})
res.render("dashboard", {title: title, id:req.params.id, someri:someri});
}
});
});
but I think this will be very bad performance-wise if I will get many entries in my DBs and I don't think this is the best way to do this.
So, if I was told correctly and I should use a pre hook for obtaining what I've said, how can I make it refer to the document?
Ok, I think I understood your requirements. this is what you could do:
/*
this will always set a documents `statusFlag` to false, if the
`dateOfIntroduction` was before Date.now()
*/
const mongoose = require('mongoose')
someriSchema.pre('find', function(next) {
mongoose.models.Somer.update(
{ datofIntroduction: { $lte: new Date() }},
{ statusFlag : false})
.exec()
.then((err, result) => {
// handle err and result
next();
});
});
The only problem I see, is that you are firing this request on every find.
in query middleware, mongoose doesn't necessarily have a reference to
the document being updated, so this refers to the query object rather
than the document being updated.
Taken straight from the documentation of mongoose
I pointed you yesterday to their documentation; but here is a more concrete answer.
someriSchema.post('find', function(res) {
// res will have all documents that were found
if (res.length > 0) {
res.forEach(function(someri){
// Do your logic of checking if 4 weeks have passed then do the following
someri.deactivated = true
someri.save()
})
}
})
What this basically do is for every found schema you would update their properties accordingly, your res can have only 1 object if you only queried 1 object. your second solution would be to do the cron
EDIT: This is what you would do to solve the async issue
const async = require('async')
someriSchema.post('find', function(res) {
async.forEach(res, function(someri, callback) {
// Do your logic of checking if 4 weeks have passed
// then do the following - or even better check if Date.now()
// is equal to expiryDate if created in the model as suggested
// by `BenSow`
// Then ONLY if the expiry is true do the following
someri.deactivated = true
someri.save(function (err) {
err ? callback(err) : callback(null)
})
}, function(err){
err ? console.log(err) : console.log('Loop Completed')
})
})

How to limit my collection size to only one document?

I have a API, which allows a user to add a intervalValue - which will be used in my program to determine how often my program runs. I am using node js, express and mongodb in my project.
This is the API which allows a user to add a value:
router.post('/', function(req, res, next) {
intervalValue.create(req.body, function(err, post) {
if (err) {
console.log(err);
} else {
res.json(post);
}
});
});
And this is the schema for it:
var intervalValueSchema = new mongoose.Schema({
milliseconds: {
type: Number,
min: 15000
}
}, {
capped: {
size: 1024,
max: 1,
autoIndexId: true
}
});
From my understanding, this schema will only allow milliseconds value larger than 15000 milliseconds, and because it is capped, it will only allow one document in the collection.
THE AIM : add a interval value, and then only be able to modify that value - i.e. it will not be allowed to be deleted, and no more will be able to be added. Hence I need it to be limited to one document.
However with the current code, I am able to add multiple documents to this collection (even though when I do isCapped() I get true returned), and when I update the value I can insert a value less than 15000 - which should not be allowed.
This is the update API:
router.put('/updateValue/:id', function(req, res, next) {
intervalPoll.findByIdAndUpdate(req.params.id, req.body, {
new: true
}, function(err, post) {
if (err) {
console.log(err);
} else {
res.json(post);
}
});
});
What am I doing wrong - I am a beginner with mongo so any tips would be appreciated. Also, is there a better way to achieve my aim?
Thanks!
I wouldn't expose the ID of that value and I would not implement the PUT handler with :id (which is not convenient anyway, because user has to know the ID which once set will never change and be only one).
I would implement two endpoints:
router.get('/interval', ...
and
router.post('/interval', ...
# or:
router.put('/interval', ...
The get handler would search the database for any document and if present return its value. If there is no such document it would return some default value.
The post handler would first verify the value and then modify a document if it exists or insert it if it doesn't.
I think that in this case it would be much easier to check for that number in your handler then to fight with Mongoose to do what you need. This is a specific use case.
Udate
Here are some examples:
Schema can be:
mongoose.Schema({
milliseconds: Number
});
And the handlers would be something like this - let's say that your model is called Interval:
var defaultInterval = {milliseconds: 15000};
router.get('/interval', function(req, res, next) {
Interval.find().exec(function (err, intervals) {
if (err || intervals.length === 0) {
res.json(defaultInterval);
} else {
res.json({milliseconds: intervals[0].milliseconds});
}
});
});
router.put('/interval', function(req, res, next) {
var milliseconds = // get it from request
if (!milliseconds || milliseconds < 15000) {
// respond with error
} else {
Interval.findOneAndUpdate(
{}, {milliseconds: milliseconds}, {upsert: true},
function(err, interval){
if (err) {
// respond with error
} else {
// respond with success
}
}
);
}
});
This is not tested and you need to fill in the blanks but you get the idea.

Sorting on _id gives inconsistent results

I have something weird, I try just to sort on _id and have some paging. Hereunder you will see the query I execute:
var condition = { isArchived: false };
if(lastId) {
condition["_id"] = { $lt : lastId };
};
PostModel
.find(condition)
.sort({_id:-1})
.limit(10)
.exec(function (err, posts) {
if(err)
return callback(new customError.Database(err.toString()),null);
callback(null, posts);
})
What I see is that in 80% of the time the result is consistent, but sometimes the result is not the same (it does not vary much, but some objects are in a different order).
I use this technique with success on other models, but only with this query/collection I get this problem (not sure if the problem is in the query or on the collection...)
What can be the reason?

Mongoose text search pagination in MongoDB/Node.js

I have this problem , i don't know how to make a pagination in Mongoose text search module, i am totally beginner, so please help, this is my code:
searchModul.create({ title: title, description: description }, function (err) {
if (err) return handleError(err);
searchModul.textSearch(title, function (err, output) {
if (err) return handleError(err);
res.render('search', {
title : 'title',
results : output.results
});
});
});
and also i want to know how to display that pagination in the search.ejs view. and thanks in advance.
To implement pagination, use $limit to define a limit for each query, and $skip to navigate pages.
From the docs:
$limit takes a positive integer that specifies the maximum number of documents to pass along.
$skip takes a positive integer that specifies the maximum number of documents to skip.
There are previous questions like this one, check out the answers here and a more detailed tutorial here.
Hope that helps!
I think you want something like this:
searchModul.find({ $text: { $search: title }}).skip(50).limit(50).exec(function (err, output) {
if (err) return handleError(err);
res.render('search', {
title : 'title',
results : output.results
});
});
This will return the second 50 items that match. .skip(50) the number of items skip and .limit(50) is the number of items to return.
const field = req.query.field;
const search = {};
search[field] = req.query.searchValue;
search["role"] = req.params._role;
search["status"] = parseInt(req.query.status);
user = await Users.paginate(search, {
limit: parseInt(req.query.limit),
page: parseInt(req.query.page),
});
OR
var ObjectId = mongoose.Types.ObjectId;
let _id = new ObjectId(req.query.q);;
user = await Users.paginate(
{
role: req.params._role,
_id: _id,
},
{
limit: parseInt(req.query.limit),
page: parseInt(req.query.page),
});

Sails.js populate nested associations

I've got myself a question regarding associations in Sails.js version 0.10-rc5. I've been building an app in which multiple models are associated to one another, and I've arrived at a point where I need to get to nest associations somehow.
There's three parts:
First there's something like a blog post, that's being written by a user. In the blog post I want to show the associated user's information like their username. Now, everything works fine here. Until the next step: I'm trying to show comments which are associated with the post.
The comments are a separate Model, called Comment. Each of which also has an author (user) associated with it. I can easily show a list of the Comments, although when I want to display the User's information associated with the comment, I can't figure out how to populate the Comment with the user's information.
In my controller i'm trying to do something like this:
Post
.findOne(req.param('id'))
.populate('user')
.populate('comments') // I want to populate this comment with .populate('user') or something
.exec(function(err, post) {
// Handle errors & render view etc.
});
In my Post's 'show' action i'm trying to retrieve the information like this (simplified):
<ul>
<%- _.each(post.comments, function(comment) { %>
<li>
<%= comment.user.name %>
<%= comment.description %>
</li>
<% }); %>
</ul>
The comment.user.name will be undefined though. If I try to just access the 'user' property, like comment.user, it'll show it's ID. Which tells me it's not automatically populating the user's information to the comment when I associate the comment with another model.
Anyone any ideals to solve this properly :)?
Thanks in advance!
P.S.
For clarification, this is how i've basically set up the associations in different models:
// User.js
posts: {
collection: 'post'
},
hours: {
collection: 'hour'
},
comments: {
collection: 'comment'
}
// Post.js
user: {
model: 'user'
},
comments: {
collection: 'comment',
via: 'post'
}
// Comment.js
user: {
model: 'user'
},
post: {
model: 'post'
}
Or you can use the built-in Blue Bird Promise feature to make it. (Working on Sails#v0.10.5)
See the codes below:
var _ = require('lodash');
...
Post
.findOne(req.param('id'))
.populate('user')
.populate('comments')
.then(function(post) {
var commentUsers = User.find({
id: _.pluck(post.comments, 'user')
//_.pluck: Retrieves the value of a 'user' property from all elements in the post.comments collection.
})
.then(function(commentUsers) {
return commentUsers;
});
return [post, commentUsers];
})
.spread(function(post, commentUsers) {
commentUsers = _.indexBy(commentUsers, 'id');
//_.indexBy: Creates an object composed of keys generated from the results of running each element of the collection through the given callback. The corresponding value of each key is the last element responsible for generating the key
post.comments = _.map(post.comments, function(comment) {
comment.user = commentUsers[comment.user];
return comment;
});
res.json(post);
})
.catch(function(err) {
return res.serverError(err);
});
Some explanation:
I'm using the Lo-Dash to deal with the arrays. For more details, please refer to the Official Doc
Notice the return values inside the first "then" function, those objects "[post, commentUsers]" inside the array are also "promise" objects. Which means that they didn't contain the value data when they first been executed, until they got the value. So that "spread" function will wait the acture value come and continue doing the rest stuffs.
At the moment, there's no built in way to populate nested associations. Your best bet is to use async to do a mapping:
async.auto({
// First get the post
post: function(cb) {
Post
.findOne(req.param('id'))
.populate('user')
.populate('comments')
.exec(cb);
},
// Then all of the comment users, using an "in" query by
// setting "id" criteria to an array of user IDs
commentUsers: ['post', function(cb, results) {
User.find({id: _.pluck(results.post.comments, 'user')}).exec(cb);
}],
// Map the comment users to their comments
map: ['commentUsers', function(cb, results) {
// Index comment users by ID
var commentUsers = _.indexBy(results.commentUsers, 'id');
// Get a plain object version of post & comments
var post = results.post.toObject();
// Map users onto comments
post.comments = post.comments.map(function(comment) {
comment.user = commentUsers[comment.user];
return comment;
});
return cb(null, post);
}]
},
// After all the async magic is finished, return the mapped result
// (or an error if any occurred during the async block)
function finish(err, results) {
if (err) {return res.serverError(err);}
return res.json(results.map);
}
);
It's not as pretty as nested population (which is in the works, but probably not for v0.10), but on the bright side it's actually fairly efficient.
I created an NPM module for this called nested-pop. You can find it at the link below.
https://www.npmjs.com/package/nested-pop
Use it in the following way.
var nestedPop = require('nested-pop');
User.find()
.populate('dogs')
.then(function(users) {
return nestedPop(users, {
dogs: [
'breed'
]
}).then(function(users) {
return users
}).catch(function(err) {
throw err;
});
}).catch(function(err) {
throw err;
);
Worth saying there's a pull request to add nested population: https://github.com/balderdashy/waterline/pull/1052
Pull request isn't merged at the moment but you can use it installing one directly with
npm i Atlantis-Software/waterline#deepPopulate
With it you can do something like .populate('user.comments ...)'.
sails v0.11 doesn't support _.pluck and _.indexBy use sails.util.pluck and sails.util.indexBy instead.
async.auto({
// First get the post
post: function(cb) {
Post
.findOne(req.param('id'))
.populate('user')
.populate('comments')
.exec(cb);
},
// Then all of the comment users, using an "in" query by
// setting "id" criteria to an array of user IDs
commentUsers: ['post', function(cb, results) {
User.find({id:sails.util.pluck(results.post.comments, 'user')}).exec(cb);
}],
// Map the comment users to their comments
map: ['commentUsers', function(cb, results) {
// Index comment users by ID
var commentUsers = sails.util.indexBy(results.commentUsers, 'id');
// Get a plain object version of post & comments
var post = results.post.toObject();
// Map users onto comments
post.comments = post.comments.map(function(comment) {
comment.user = commentUsers[comment.user];
return comment;
});
return cb(null, post);
}]
},
// After all the async magic is finished, return the mapped result
// (or an error if any occurred during the async block)
function finish(err, results) {
if (err) {return res.serverError(err);}
return res.json(results.map);
}
);
You could use async library which is very clean and simple to understand. For each comment related to a post you can populate many fields as you want with dedicated tasks, execute them in parallel and retrieve the results when all tasks are done. Finally, you only have to return the final result.
Post
.findOne(req.param('id'))
.populate('user')
.populate('comments') // I want to populate this comment with .populate('user') or something
.exec(function (err, post) {
// populate each post in parallel
async.each(post.comments, function (comment, callback) {
// you can populate many elements or only one...
var populateTasks = {
user: function (cb) {
User.findOne({ id: comment.user })
.exec(function (err, result) {
cb(err, result);
});
}
}
async.parallel(populateTasks, function (err, resultSet) {
if (err) { return next(err); }
post.comments = resultSet.user;
// finish
callback();
});
}, function (err) {// final callback
if (err) { return next(err); }
return res.json(post);
});
});
As of sailsjs 1.0 the "deep populate" pull request is still open, but the following async function solution looks elegant enough IMO:
const post = await Post
.findOne({ id: req.param('id') })
.populate('user')
.populate('comments');
if (post && post.comments.length > 0) {
const ids = post.comments.map(comment => comment.id);
post.comments = await Comment
.find({ id: commentId })
.populate('user');
}
Granted this is an old question, but a much simpler solution would be to loop over the comments,replacing each comment's 'user' property (which is an id) with the user's full detail using async await.
async function getPost(postId){
let post = await Post.findOne(postId).populate('user').populate('comments');
for(let comment of post.comments){
comment.user = await User.findOne({id:comment.user});
}
return post;
}
Hope this helps!
In case anyone is looking to do the same but for multiple posts, here's one
way of doing it:
find all user IDs in posts
query all users in 1 go from DB
update posts with those users
Given that same user can write multiple comments, we're making sure we're reusing those objects. Also we're only making 1 additional query (whereas if we'd do it for each post separately, that would be multiple queries).
await Post.find()
.populate('comments')
.then(async (posts) => {
// Collect all comment user IDs
const userIDs = posts.reduce((acc, curr) => {
for (const comment of post.comments) {
acc.add(comment.user);
}
return acc;
}, new Set());
// Get users
const users = await User.find({ id: Array.from(userIDs) });
const usersMap = users.reduce((acc, curr) => {
acc[curr.id] = curr;
return acc;
}, {});
// Assign users to comments
for (const post of posts) {
for (const comment of post.comments) {
if (comment.user) {
const userID = comment.user;
comment.user = usersMap[userID];
}
}
}
return posts;
});

Resources