My server need to handle too many requests (ex:100 req/sec) at a time and it involves db(mongodb) operations. To accomplish that, I've chosen Express Server to handle those request. I set up server with the express. Here I want to create only one db-connection for entire server.
To achieve it:
in app.js
var mongo = require('mongodb');
var monk = require('monk');
var db = monk('localhost:27017/testdb1');
app.use(function (req, res, next) {
req.db1 = db;
//console.log(req.db1);
next();
});
in routes.js
router.get('/user', function (req, res) {
if (req) {
find = {};
var db1 = req.db1;
var userCollection = db1.get(collections.user);
var testCollection1 = db1.get(collections.testTable1);
var testCollection2 = db1.get(collections.testTable2);
userCollection.find(find, function (err, doc) {
console.log('1');
});
testCollection1.find(find, function (err, doc) {
console.log('2');
});
testCollection2.find(find, function (err, doc) {
console.log('3');
});
}
});
And the Problem is:
Let current mongodb connections(cmc) = 1;
server creates one connection as it lifts(cmc++). When I make a request to route(/user) another connection is created(cmc++) for the first time. When I make a request again, one more connection is created(cmc++). After that although I made a number of requests to that route the count remains the same(cmc=4). What exactly happening there ?
I need to update more collections when request is made. As no.of operations on collections increases connection count increasing.
why the connection count is increasing. I want to create only one connection.
Related
I am using the express framework and would like to connect to a mongodb without using mongoose, but with the native nodejs Mongodb driver. How can I do this without creating a new connection every time?
To handle get or post requests I currently open a new connection to the db for every request and close it on completion of the request. Is there a better way to do this? Thanks in advance.
Following the example from my comment, modifying it so that the app handles errors rather than failing to start the server.
var express = require('express');
var mongodb = require('mongodb');
var app = express();
var MongoClient = require('mongodb').MongoClient;
var dbURL = "mongodb://localhost:27017/integration_test";
var db;
// Initialize connection once
MongoClient.connect(dbURL, function(err, database) {
if(err) return console.error(err);
db = database;
// the Mongo driver recommends starting the server here
// because most apps *should* fail to start if they have no DB.
// If yours is the exception, move the server startup elsewhere.
});
// Reuse database object in request handlers
app.get("/", function(req, res, next) {
var collection = "replicaset_mongo_client_collection";
db.collection(collection).find({}, function(err, docs) {
if(err) return next(err);
docs.each(function(err, doc) {
if(doc) {
console.log(doc);
}
else {
res.end();
}
});
});
});
app.use(function(err, req, res){
// handle error here. For example, logging and
// returning a friendly error page
});
// Starting the app here will work, but some users
// will get errors if the db connection process is slow.
app.listen(3000);
console.log("Listening on port 3000");
var mongodb = require('mongodb');
var uri = 'mongodb://localhost:27017/dbname';
module.exports = function(callback) {
mongodb.MongoClient.connect(uri, callback);
};
Ad this snippet in a file say connect.js and then require this file(connect.js) in your file where you are declaring your functions for http requests.
The index.js file:
var express = require('express');
var router = express.Router();
function validCheck (exp,name) {
return exp.test(name);
}
/* GET home page. */
router.get('/', function(req, res, next) {
res.render('index');
});
var user=db.collection('user'); //gives reference error,db is not defined
router.post('/',function(req,res,next){
username=req.body.username;
password=req.body.password;
//var user=db.collection('user'); //works fine
user.findOne({'username':username,'password':password},function(err,docs){
//do something
});
});
module.exports = router;
When using var user=db.collection('user') outside router.post, it gives the error but when the same is used inside router.post, it works fine.
what might be the concept I am missing here?
Edit: part of App.js file involving mongodb
var mongodb= require('mongodb');
var MongoClient= mongodb.MongoClient;
var URL = 'mongodb://127.0.0.1:27017/mainDB';
MongoClient.connect(URL,function(err,database){
if(!err){
db=database;
}
else{
//do something
}
});
Remember MongoClient.connect() is async. The database connection might not be ready at the time you do var user=db.collection('user');. The database connection has been made one the callback is done, no earlier than that.
When the first request is done, the database connection just happens to be established. The longer you wait the more likely it is that is works, but still it's the wrong approach.
Also working with global variables is bad practice and leads to confusion and other problems.
In short the code should look like
// module database.js
var mongodb= require('mongodb');
var MongoClient= mongodb.MongoClient;
var URL = 'mongodb://127.0.0.1:27017/mainDB';
var db;
var error;
var waiting = []; // Callbacks waiting for the connection to be made
MongoClient.connect(URL,function(err,database){
error = err;
db = database;
waiting.forEach(function(callback) {
callback(err, database);
});
});
module.exports = function(callback) {
if (db || error) {
callback(error, db);
} else {
waiting.push(callback);
}
}
}
Than use it like
var db = require('database.js');
router.post('/',function(req,res,next){
username=req.body.username;
password=req.body.password;
db.conn(function(err, database) {
if (err) {
res.sendStatus(500);
console.log(err);
return;
}
database.collection('users').findOne({'username':username, 'password':password}, function(err, docs){
//do something
});
});
});
Note that the connection is made on the first require, so if you add require('database.js'); in App.js. You don't loose the on the first request.
Alternatively you can use promises, which takes care of the waiting logic for you.
What you are missing is the timing of execution.
The var user line outside router.post is executed immediately when index.js is processed by node.js.
The var user line inside router.post is only executed when a client requests the / page of your app.
The db variable is only assigned after the connection to your MongoDB has been made successfully. This is too late for the first var user line, but in time for the first HTTP request to /.
MongoClient.connect's callback function is called asynchronously only after it is connected to the database. At the time only you are defining db as a global variable.
Assuming you require index.js file in app.js. The line var user=db.collection('user'); outside router.post is executed synchronously. By the time this line is executed db would have not be defined.
The line var user=db.collection('user'); inside router.post is executed asynchronously. So, when that code is executed db is already defined.
To your question:
what might be the concept I am missing here?
I suggest you to learn about asynchronous javascript execution https://stackoverflow.com/a/7104633/3492210
var mongo = require('mongodb');
var url = "mongodb://localhost:27017/qanda";
mongo.MongoClient.connect(url, function(err, db) {
if (err) throw err;
console.log("Database created!");
db.close();
});
I have an http server and every time it gets a post request, it is supposed to insert the data into MongoDB. This server is supposed to be constantly running and accepting thousands of request in any given second.
How can I maximize the efficiency and speed of my code? I feel like my current code is not fast enough and furthermore wastes CPU power when it makes a new db every time it receives a request.
My current layout
var server = http.createServer(function (req, res) {
req.on('data', function(chunk) {
//Receive my data
});
req.on('end', function() {
//JSON parse my data
var db = new Db('test', new Server("111.111.11.111", 27017,{auto_reconnect: false}), {safe: true});
db.open(function(err, db) {
//Insert data into DB
db.close();
});
});
}); //End Http server
server.listen(8999);
I have tried replacing db.open with MongoClient.connect, but that considerably slows down processing and I don't know why. In this case, the older version of MongoDB Native for node js seems to work faster.
You'll want to shift to an approach where you open a large pool of connections during startup that are shared by your HTTP request handlers. To tweak the MongoDB connection pool size to suit whatever scalability needs you have, pass an options parameter to your MongoClient.connect call.
var options = {
server: {
// The number of pooled connection available to your app.
poolSize: 100
}
};
mongodb.MongoClient.connect('mongodb://111.111.11.111/test', options,
function(err, db) {
var server = http.createServer(function (req, res) {
// Your req.on calls go here, directly using db rather than creating
// new instances. Don't close db either.
});
server.listen(8999);
}
);
Not sure if this would be better, but you can encapsulate that server inside the db, therefore persisting the connection:
var db = new Db('test', new Server("111.111.11.111", 27017,{auto_reconnect: false}), {safe: true});
db.open(function(err, db) {
var server = http.createServer(function (req, res) {
//now do stuff thru constant open connection
});
db.close();
});
On my local host, I have the following Node code to setup a mongoDB database name "dbname":
users.js:
var MongoClient = require("mongodb").MongoClient,
Connection = require("mongodb").Connection,
Server = require("mongodb").Server;
Users = function(host, port) {
var mongoClient = new MongoClient(new Server(host, port));
mongoClient.open(function (){});
this.db = mongoClient.db("dbname");
};
Users.prototype.getCollection = function (callback) {
this.db.collection("users", function (error, users) {
if (error) callback(error);
else callback(null, users);
});
};
Users.prototype.findAll = function (callback) {
this.getCollection(function (error, users) {
if (error) {
callback(error);
} else {
users.find().toArray(function (error, results) {
if (error) {
callback(error);
} else {
callback(null,results);
}
});
}
});
}
// Bunch of other prototype functions...
exports.Users = Users;
I like to put the above database functionality in one file, and then in my main server file require that file as follows:
server.js:
var Users = require("./users").Users;
var users = new Users("localhost", 27017);
users.findAll(function (err, user) {
// Do something
});
To have this working on localhost is pretty easy. In the command line, I just type the following:
$ mongod # to launch the database server
$ node server.js # to launch the web server
and it works fine. However, now I'm trying to push the whole thing onto Heroku with the mongolab addon
heroku addons:add mongolab
but the database is not running and I have no idea how to make it run. This tutorial explains how to setup mongodb with the mongolab URI, but that's not how my code works, I use a host and a port and I create a new server based on that. How should I change my code for it to work on the heroku app? I want to keep the database code in a separate file, with the prototype functions.
Follow the example here at the "MongoClient.connect" section.
Essentially, you will need to change this part of the code:
Users = function(host, port) {
var mongoClient = new MongoClient(new Server(host, port));
mongoClient.open(function (){});
this.db = mongoClient.db("dbname");
};
To use mongoClient.connect() instead of new MongoClient:
Users = function(url) {
MongoClient.connect(url, function(err, db) {
// Find better way to set this since this callback is asynchronous.
this.db = db;
});
};
If you are using node, I recommend using a library such as mongoose npm install mongoose to handle mongodb interactions. Look at my answer here for how to structure your schemas.
Helped by Xinzz's answer, here's the modified code, so that the mongodb database is initialized with a URI instead of host + port. That's how Heroku initializes the mongodb database, and that's why it wasn't working.
var mongodb = require("mongodb");
var MONGODB_URI = process.env.MONGOLAB_URI || process.env.MONGOHQ_URL || "mongodb://localhost", // Make sure to replace that URI with the one provided by MongoLab
db,
users;
mongodb.MongoClient.connect(MONGODB_URI, function (err, database) {
if (err) throw err;
db = database;
users = db.collection("users");
accounts = db.collection("accounts");
var server = app.listen(process.env.PORT || 3000);
console.log("Express server started on port %s", server.address().port);
});
The key here is to declare the variables db and users upfront, assign them a value in the asynchronous callback of the connect function of MongoClient and also start the app (app.listen(...)) in the same callback. Then later in the code I can do the following:
users.find().toArray(function (err, results) {
// Do something
});
I also gave up on all these prototype functions, since they did not really add much.
I am currently rolling back from mongoose to node-mongodb-native.
So I am quite new at this topic. However my issue currently is that want to create a database collection on server start which I then can use through the application. Unfortunately I only find examples in the repository where you only can do database actions directly in the callback of the connect function.
docs:
var mongodb = require("mongodb"),
mongoServer = new mongodb.Server('localhost', 27017),
dbConnector = new mongodb.Db('example', mongoServer);
db_connector.open(function(err, db) {
if (err) throw new Error(err);
// here I can do my queries etc.
});
But how can I get access to the db object in the callback when I am in some route callback?
Currently the only idea I would have is wrapping the application into the callback:
var mongodb = require("mongodb"),
express = require("express"),
mongoServer = new mongodb.Server('localhost', 27017),
dbConnector = new mongodb.Db('example', mongoServer);
var app = new express();
db_connector.open(function(err, db) {
if (err) throw new Error(err);
app.get('/products', function(req, res, next) {
db.collection('products', function(err, collection) {
if (err) next(new Error(err));
collection.find({}, function(err, products) {
res.send(products);
});
});
});
});
But I do not think this is the way it should meant to be?
Isn't there the way to create a sync database connection call which I then can easily use through the whole application how it was by mongoose?
Regards bodo
Db.open opens the connection to mongodb and returns a reference to itself. See here for the sourcecode: https://github.com/mongodb/node-mongodb-native/blob/master/lib/mongodb/db.js#L245
All you want is to hold off on starting your express app listening on it's port and receiving requests until your connection to mongodb has been established.
So what you could do is this:
var mongodb = require("mongodb"),
express = require("express"),
mongoServer = new mongodb.Server('localhost', 27017),
dbConnector = new mongodb.Db('example', mongoServer),
db;
var app = new express();
app.get('/products', function(req, res, next) {
db.collection('products', function(err, collection) {
if (err) next(new Error(err));
collection.find({}, function(err, products) {
res.send(products);
});
});
});
db_connector.open(function(err, opendb) {
if (err) throw new Error(err);
db = opendb;
app.listen(3000);
});
What I'm not sure about though is whether this is a good idea. This solution doesn't allow you to recreate your connection if there has been a connection break or you restarted your mongodb process. So while the above might work, it might be a better idea to create a method that will wrap the creation of a connection to mongodb.