Our collection has around 100 million documents. We created a simple application using nodejs and expressjs with a limit clause in the mongo query . It is sufficient for the users as of now. In the mean time we are trying to implement lazy loading so that the initial page load returns few documents and when the user scrolls we would like to load further documents. Struggling where to start and how to ahead to implement it. Appreciate your suggestions.
My index.js file would look like this
router.get('/users', function(req, res) {
var db = req.db;
var users = db.get('users');
users.find(query, {limit: 10000}, function(e, docs){
res.render('users', {
title: 'Users',
'users': docs
});
});
});
I would like to remove the limit and using skip trying to achieve this.
Please post your suggestions
This should help. It uses the .skip method of the .find() cursor. I call it pagination rather than lazy loading.
var itemsPerPage = 10;
router.get('/users/:pageNum', function(req, res) {
var db = req.db;
var users = db.get('users');
users.find(query, {skip: (itemsPerPage * (pageNum-1)), limit: itemsPerPage},function(e, docs){
res.render('users', {
title: 'Users',
'users': docs
});
});
});
You could implement paging in your route
Something like below;
// GET /users?start=20
router.get('/users', function(req, res) {
var db = req.db;
var start = parseInt(req.query.start) || 0;
var users = db.get('users');
users.find(query, {start: start, limit: 10000}, function(e, docs){
res.render('users', {
title: 'Users',
'users': docs
});
});
});
I suggest you use Mongoose to model the MongoDB objects efficiently and then make use of one of many mongoose pagination modules available.
Related
I am very new to Node.js and MongoDB and am trying to piece together my own blogging application. I have a problem trying to query through my 'Blog' model for ones with a specific username. When I try to run:
var userBlogs = function(username) {
ub = Blog.find({author: username}).toArray();
ub = ub.reverse();
};
I get an error:
TypeError: Object #<Query> has no method 'toArray'
I know globals are bad but I've just been trying to get it to work. The Mongo documentation claims that a cursor is returned which can have the toArray() method called on it. I have no idea why it won't work.
Here is my schema/model creation:
var blogSchema = mongoose.Schema({
title: {type:String, required: true},
author: String,
content: {type:String, required: true},
timestamp: String
});
var Blog = mongoose.model('Blog', blogSchema);
Here are the /login and /readblog requests
app.get('/readblog', ensureAuthenticated, function(req, res) {
res.render('readblog', {user: req.user, blogs: ub})
})
app.get('/login', function(req, res){
res.render('login', { user: req.user, message: req.session.messages });
});
app.post('/login',
passport.authenticate('local', { failureRedirect: '/login'}),
function(req, res) {
userBlogs(req.user.username);
res.redirect('/');
});
});
The end result is supposed to work with this Jade:
extends layout
block content
if blogs
for blog in blogs
h2= blog[title]
h4= blog[author]
p= blog[content]
h4= blog[timestamp]
a(href="/writeblog") Write a new blog
How can I get the query to output an array, or even work as an object?
The toArray function exists on the Cursor class from the Native MongoDB NodeJS driver (reference). The find method in MongooseJS returns a Query object (reference). There are a few ways you can do searches and return results.
As there are no synchronous calls in the NodeJS driver for MongoDB, you'll need to use an asynchronous pattern in all cases. Examples for MongoDB, which are often in JavaScript using the MongoDB Console imply that the native driver also supports similar functionality, which it does not.
var userBlogs = function(username, callback) {
Blog.find().where("author", username).
exec(function(err, blogs) {
// docs contains an array of MongooseJS Documents
// so you can return that...
// reverse does an in-place modification, so there's no reason
// to assign to something else ...
blogs.reverse();
callback(err, blogs);
});
};
Then, to call it:
userBlogs(req.user.username, function(err, blogs) {
if (err) {
/* panic! there was an error fetching the list of blogs */
return;
}
// do something with the blogs here ...
res.redirect('/');
});
You could also do sorting on a field (like a blog post date for example):
Blog.find().where("author", username).
sort("-postDate").exec(/* your callback function */);
The above code would sort in descending order based on a field called postDate (alternate syntax: sort({ postDate: -1}).
Try something along the lines of:
Blog.find({}).lean().exec(function (err, blogs) {
// ... do something awesome...
}
You should utilize the callback of find:
var userBlogs = function(username, next) {
Blog.find({author: username}, function(err, blogs) {
if (err) {
...
} else {
next(blogs)
}
})
}
Now you can get your blogs calling this function:
userBlogs(username, function(blogs) {
...
})
I just start to use node.js with express and mongoose, and I have a stupid question...
Somewhere in my routes.js file, I have the following section :
// DASHBOARD SECTION, GET MY GROUPS
app.get('/dashboard', isLoggedIn, function(req, res) {
var Group = require('../app/models/group'); //adding mongoose Schema
Group.find({"groupDetails.userId" : req.user._id})
.exec(function(err, myGroups) {
if (err)
res.send(err);
var myGroups = myGroups;
//console.log("myGroups: " + myGroups); // check with "heroku logs"
res.render('dashboard.ejs', {
user : req.user,
myGroups : myGroups
});
});
});
This code works. When someone browse the dashboard page, I receive "myGroups" which is an array with all the groups for the current logged in user.
Now, here is my question :
Actually when someone browse the dashboard page, I would like to make a second query (based on the exact same pattern) to get all groups and all files for the current logged in user.
Then I will send "user", "myGroups" and "myFiles" to the dashboard page.
How can I do that ?
I tried several things with no result so far... I think I'm a little bit lost in node.js callback functions :D
Thanks a lot for your help.
You have two options here:
1) deal with callback hell (callback inside callback inside...) to retrieve 3 sets of data. This way is least elegant and efficient
2) Use a library that will do the job asynchronously and have one callback when all the data is retrieved, you can use async library which is just awesome. In this case you will have just one callback in which you can access all the data you have fetched.
Here's what you can do with async in your case:
var async = require('async');
..........
app.get('/dashboard', isLoggedIn, function(req, res) {
var Group = require('../app/models/group'); //adding mongoose Schema
var User = require('../app/models/user'); //adding mongoose Schema
var Files = require('../app/models/files'); //adding mongoose Schema
async.parallel({
groups: function(callback){
Group.find(...).exec(callback);
},
users: function(callback){
Users.find(...).exec(callback);
},
files: function(callback){
Files.find(...).exec(callback);
}
}, function(err, results) {
if (err)
res.send(err);
var groups = results.groups;
var users = results.users;
var files = results.files;
res.render('dashboard.ejs', {
user : req.user,
myGroups : groups,
users: users,
files: files
});
});
});
I see lots of examples where node.js / express router code is organized like this:
// server.js
var cats = require('cats');
app.get('/cats', cats.findAll);
// routes/cats.js
exports.findAll = function(req, res) {
// Lookup all the cats in Mongoose CatModel.
};
I'm curious if it would be okay to put the logic to create, read, update and delete cats in the mongoose CatModel as methods? So you could do something like cat.findAll(); The model might look something like this:
var Cat = new Schema({
name: {
type: String,
required: true
}
});
Cat.methods.findAll = function(callback) {
// find all cats.
callback(results);
}
Then you could use this in your router:
app.get('/cats', cats.findAll);
If if further logic / abstraction is needed (to process the results) then one could do it in routes/cats.js.
Thanks in Advance.
Obviously your architecture is completely up to you. I've found that separating my routes (which handle business logic) and models (which interact with the db) is necessary and very easy.
So I would usually have something like
app.js
var cats = require ('./routes/cats');
app.get('/api/cats', cats.getCats);
routes/cats.js
var Cats = require ('../lib/Cats');
exports.getCats = function (req, res, next) {
Cat.get (req.query, function (err, cats) {
if (err) return next (err);
return res.send ({
status: "200",
responseType: "array",
response: cats
});
});
};
lib/Cat.js
var catSchema = new Schema({
name: {
type: String,
required: true
}
});
var Cat = mongoose.model ('Cat', catSchema);
module.exports = Cat;
Cat.get = function (params, cb) {
var query = Cat.find (params);
query.exec (function (err, cats) {
if (err) return cb (err);
cb (undefined, cats);
});
};
So this example doesn't exactly show an advantage, but if you had an addCat route, then the route could use a "getCatById" function call, verify the cat doesn't exist, and add it. It also helps with some nesting. The routes could also be used for sanitizing the objects before sending them off, and might also send resources and information used in UI that isn't necessarily coupled with mongoose. It also allows interactions with the database to be reusable in multiple routes.
I am very new to Node.js and MongoDB and am trying to piece together my own blogging application. I have a problem trying to query through my 'Blog' model for ones with a specific username. When I try to run:
var userBlogs = function(username) {
ub = Blog.find({author: username}).toArray();
ub = ub.reverse();
};
I get an error:
TypeError: Object #<Query> has no method 'toArray'
I know globals are bad but I've just been trying to get it to work. The Mongo documentation claims that a cursor is returned which can have the toArray() method called on it. I have no idea why it won't work.
Here is my schema/model creation:
var blogSchema = mongoose.Schema({
title: {type:String, required: true},
author: String,
content: {type:String, required: true},
timestamp: String
});
var Blog = mongoose.model('Blog', blogSchema);
Here are the /login and /readblog requests
app.get('/readblog', ensureAuthenticated, function(req, res) {
res.render('readblog', {user: req.user, blogs: ub})
})
app.get('/login', function(req, res){
res.render('login', { user: req.user, message: req.session.messages });
});
app.post('/login',
passport.authenticate('local', { failureRedirect: '/login'}),
function(req, res) {
userBlogs(req.user.username);
res.redirect('/');
});
});
The end result is supposed to work with this Jade:
extends layout
block content
if blogs
for blog in blogs
h2= blog[title]
h4= blog[author]
p= blog[content]
h4= blog[timestamp]
a(href="/writeblog") Write a new blog
How can I get the query to output an array, or even work as an object?
The toArray function exists on the Cursor class from the Native MongoDB NodeJS driver (reference). The find method in MongooseJS returns a Query object (reference). There are a few ways you can do searches and return results.
As there are no synchronous calls in the NodeJS driver for MongoDB, you'll need to use an asynchronous pattern in all cases. Examples for MongoDB, which are often in JavaScript using the MongoDB Console imply that the native driver also supports similar functionality, which it does not.
var userBlogs = function(username, callback) {
Blog.find().where("author", username).
exec(function(err, blogs) {
// docs contains an array of MongooseJS Documents
// so you can return that...
// reverse does an in-place modification, so there's no reason
// to assign to something else ...
blogs.reverse();
callback(err, blogs);
});
};
Then, to call it:
userBlogs(req.user.username, function(err, blogs) {
if (err) {
/* panic! there was an error fetching the list of blogs */
return;
}
// do something with the blogs here ...
res.redirect('/');
});
You could also do sorting on a field (like a blog post date for example):
Blog.find().where("author", username).
sort("-postDate").exec(/* your callback function */);
The above code would sort in descending order based on a field called postDate (alternate syntax: sort({ postDate: -1}).
Try something along the lines of:
Blog.find({}).lean().exec(function (err, blogs) {
// ... do something awesome...
}
You should utilize the callback of find:
var userBlogs = function(username, next) {
Blog.find({author: username}, function(err, blogs) {
if (err) {
...
} else {
next(blogs)
}
})
}
Now you can get your blogs calling this function:
userBlogs(username, function(blogs) {
...
})
I am using
app.get('/', function(req, res){
var listData = function(err, collection) {
collection.find().toArray(function(err, results) {
res.render('index.html', { layout : false , 'title' : 'Monode-crud', 'results' : results });
});
}
var Client = new Db('monode-crud', new Server('127.0.0.1', 27017, {}));
Client.open(function(err, pClient) {
Client.collection('users', listData);
//Client.close();
});
})
This method to get records from mongoDB and showing in a html.
I am using Express, hbs as view engine.
I need a common method where I can pass collection name or mongo query,
it should return the collection as result.
In the main page i can process the collection.
Just like splitting data logic in separate classes in C#.
How to achieve this?