I am using the express framework and would like to connect to a mongodb without using mongoose, but with the native nodejs Mongodb driver. How can I do this without creating a new connection every time?
To handle get or post requests I currently open a new connection to the db for every request and close it on completion of the request. Is there a better way to do this? Thanks in advance.
Following the example from my comment, modifying it so that the app handles errors rather than failing to start the server.
var express = require('express');
var mongodb = require('mongodb');
var app = express();
var MongoClient = require('mongodb').MongoClient;
var dbURL = "mongodb://localhost:27017/integration_test";
var db;
// Initialize connection once
MongoClient.connect(dbURL, function(err, database) {
if(err) return console.error(err);
db = database;
// the Mongo driver recommends starting the server here
// because most apps *should* fail to start if they have no DB.
// If yours is the exception, move the server startup elsewhere.
});
// Reuse database object in request handlers
app.get("/", function(req, res, next) {
var collection = "replicaset_mongo_client_collection";
db.collection(collection).find({}, function(err, docs) {
if(err) return next(err);
docs.each(function(err, doc) {
if(doc) {
console.log(doc);
}
else {
res.end();
}
});
});
});
app.use(function(err, req, res){
// handle error here. For example, logging and
// returning a friendly error page
});
// Starting the app here will work, but some users
// will get errors if the db connection process is slow.
app.listen(3000);
console.log("Listening on port 3000");
var mongodb = require('mongodb');
var uri = 'mongodb://localhost:27017/dbname';
module.exports = function(callback) {
mongodb.MongoClient.connect(uri, callback);
};
Ad this snippet in a file say connect.js and then require this file(connect.js) in your file where you are declaring your functions for http requests.
Related
I am getting started with mongoDB and I have to say that the official documentation is not that great to see how to implement it with nodejs.
I don't really know how to structure my server file to add mongoClient.connect, should my whole server be written inbeetwen the mongoClient.connect function in order to have access to the db, like in this boilerplate? I am using nodeJS/express.
If you know any good boilerplate, or anything, that could show me the structure of a backend with an implementation of mongoDB, I would really appreciate it. Every time I find something about mongoDB, it is actually about mongooooose!!
After further reasearch, here is what I was looking for, for those who wonder like me how to implement MongoDB (and not mongoose) with Express:
var express = require('express');
var mongodb = require('mongodb');
var app = express();
var MongoClient = require('mongodb').MongoClient;
var db;
// Initialize connection once
MongoClient.connect("mongodb://localhost:27017/integration_test", function(err, database) {
if(err) throw err;
db = database;
// Start the application after the database connection is ready
app.listen(3000);
console.log("Listening on port 3000");
});
// Reuse database object in request handlers
app.get("/", function(req, res) {
db.collection("replicaset_mongo_client_collection").find({}, function(err, docs) {
docs.each(function(err, doc) {
if(doc) {
console.log(doc);
}
else {
res.end();
}
});
});
});
I've found several ways of doing it, even in mongoDB's official pages.
By far, I prefer this one (not mine, source below) where you instantiate the connection in one file and export it and the database/client to the server file where express is instantiated:
(I copied only what's important, without error handling)
// database.js
const MongoClient = require('mongodb').MongoClient;
let _db; //'_' private
const mongoConnect = function(callback) {
MongoClient.connect(
'mongodb://localhost:27017',
{ useUnifiedTopology: true }
)
.then(client => {
_db = client.db('onlineshopping');
callback();
})
.catch(error => {
console.log(err);
throw new Error('DB connection failed...');
});
}
const getDB = () => {
if (_db) {
return _db;
} else {
throw new Error('DB connect failed');
}
}
exports.mongoConnect = mongoConnect;
exports.getDB = getDB;
// index.js
const express = require('express');
const app = express();
const mongoConnect = require('./util/database').mongoConnect;
// ...
mongoConnect(() => {
app.listen(3000);
})
Source:
https://github.com/TinaXing2012/nodejs_examples/blob/master/day9/util/database.js
Corresponding to this YouTube course that I recommend in this topic: https://www.youtube.com/watch?v=hh-gK0_HLEY&list=PLGTrAf5-F1YLBTY1mToc_qyOiZizcG_LJ&index=98
Other alternatives from mongoDB official repos, are:
https://github.com/mongodb-developer/mern-stack-example
https://github.com/mongodb-developer/nodejs-quickstart
I am using node-run-cmd package to start the mongodb server in my app.js file. I know this works because I can see the collections on Robomongo when my script is running as well as the mongod.exe in my list of running processes. The problem is trying to connect to the db called testDB. Below is the commented code.
//start mongodb server
//this works
var nrc = require('node-run-cmd');
var cmd = '..\\mongodb\\mongod --dbpath '+__dirname+'\\db';
nrc.run(cmd);
//setup
var express = require('express');
var MongoClient = require('mongodb').MongoClient;
var app = express();
app.use(express.static('public'));
app.set('view engine', 'ejs');
//connect to mongo
//this fails to connect to db testDB
var url = 'mongodb://localhost:27017/testDB';
MongoClient.connect(url, function(err, db) {
if(!err) {
console.log("connection successful");
}
else{
console.log(err.message)
}
});
Here is the err.message
failed to connect to server [localhost:27017] on first connect
Any idea what I am doing wrong here. My assumption is that the db connection is trying before the server has fully started but I am not completely sure.
EDIT:
so that's what it was, timing issue. I tried the following and it connected to the DB. Is there a graceful way of doing this other than what I have here?
function connect(){
var url = 'mongodb://localhost:27017/testDB';
MongoClient.connect(url, function(err, db) {
if (!err) {
console.log("connection successful");
}
else {
console.log(err.message)
}
});
}
setTimeout(connect, 10000);
You should use the callback in the node_run_cmd package (https://www.npmjs.com/package/node-run-cmd#callback-style).
Place your connect function inside the callback.
You will probably also want to only start express here as well.
My server need to handle too many requests (ex:100 req/sec) at a time and it involves db(mongodb) operations. To accomplish that, I've chosen Express Server to handle those request. I set up server with the express. Here I want to create only one db-connection for entire server.
To achieve it:
in app.js
var mongo = require('mongodb');
var monk = require('monk');
var db = monk('localhost:27017/testdb1');
app.use(function (req, res, next) {
req.db1 = db;
//console.log(req.db1);
next();
});
in routes.js
router.get('/user', function (req, res) {
if (req) {
find = {};
var db1 = req.db1;
var userCollection = db1.get(collections.user);
var testCollection1 = db1.get(collections.testTable1);
var testCollection2 = db1.get(collections.testTable2);
userCollection.find(find, function (err, doc) {
console.log('1');
});
testCollection1.find(find, function (err, doc) {
console.log('2');
});
testCollection2.find(find, function (err, doc) {
console.log('3');
});
}
});
And the Problem is:
Let current mongodb connections(cmc) = 1;
server creates one connection as it lifts(cmc++). When I make a request to route(/user) another connection is created(cmc++) for the first time. When I make a request again, one more connection is created(cmc++). After that although I made a number of requests to that route the count remains the same(cmc=4). What exactly happening there ?
I need to update more collections when request is made. As no.of operations on collections increases connection count increasing.
why the connection count is increasing. I want to create only one connection.
The index.js file:
var express = require('express');
var router = express.Router();
function validCheck (exp,name) {
return exp.test(name);
}
/* GET home page. */
router.get('/', function(req, res, next) {
res.render('index');
});
var user=db.collection('user'); //gives reference error,db is not defined
router.post('/',function(req,res,next){
username=req.body.username;
password=req.body.password;
//var user=db.collection('user'); //works fine
user.findOne({'username':username,'password':password},function(err,docs){
//do something
});
});
module.exports = router;
When using var user=db.collection('user') outside router.post, it gives the error but when the same is used inside router.post, it works fine.
what might be the concept I am missing here?
Edit: part of App.js file involving mongodb
var mongodb= require('mongodb');
var MongoClient= mongodb.MongoClient;
var URL = 'mongodb://127.0.0.1:27017/mainDB';
MongoClient.connect(URL,function(err,database){
if(!err){
db=database;
}
else{
//do something
}
});
Remember MongoClient.connect() is async. The database connection might not be ready at the time you do var user=db.collection('user');. The database connection has been made one the callback is done, no earlier than that.
When the first request is done, the database connection just happens to be established. The longer you wait the more likely it is that is works, but still it's the wrong approach.
Also working with global variables is bad practice and leads to confusion and other problems.
In short the code should look like
// module database.js
var mongodb= require('mongodb');
var MongoClient= mongodb.MongoClient;
var URL = 'mongodb://127.0.0.1:27017/mainDB';
var db;
var error;
var waiting = []; // Callbacks waiting for the connection to be made
MongoClient.connect(URL,function(err,database){
error = err;
db = database;
waiting.forEach(function(callback) {
callback(err, database);
});
});
module.exports = function(callback) {
if (db || error) {
callback(error, db);
} else {
waiting.push(callback);
}
}
}
Than use it like
var db = require('database.js');
router.post('/',function(req,res,next){
username=req.body.username;
password=req.body.password;
db.conn(function(err, database) {
if (err) {
res.sendStatus(500);
console.log(err);
return;
}
database.collection('users').findOne({'username':username, 'password':password}, function(err, docs){
//do something
});
});
});
Note that the connection is made on the first require, so if you add require('database.js'); in App.js. You don't loose the on the first request.
Alternatively you can use promises, which takes care of the waiting logic for you.
What you are missing is the timing of execution.
The var user line outside router.post is executed immediately when index.js is processed by node.js.
The var user line inside router.post is only executed when a client requests the / page of your app.
The db variable is only assigned after the connection to your MongoDB has been made successfully. This is too late for the first var user line, but in time for the first HTTP request to /.
MongoClient.connect's callback function is called asynchronously only after it is connected to the database. At the time only you are defining db as a global variable.
Assuming you require index.js file in app.js. The line var user=db.collection('user'); outside router.post is executed synchronously. By the time this line is executed db would have not be defined.
The line var user=db.collection('user'); inside router.post is executed asynchronously. So, when that code is executed db is already defined.
To your question:
what might be the concept I am missing here?
I suggest you to learn about asynchronous javascript execution https://stackoverflow.com/a/7104633/3492210
var mongo = require('mongodb');
var url = "mongodb://localhost:27017/qanda";
mongo.MongoClient.connect(url, function(err, db) {
if (err) throw err;
console.log("Database created!");
db.close();
});
I am currently rolling back from mongoose to node-mongodb-native.
So I am quite new at this topic. However my issue currently is that want to create a database collection on server start which I then can use through the application. Unfortunately I only find examples in the repository where you only can do database actions directly in the callback of the connect function.
docs:
var mongodb = require("mongodb"),
mongoServer = new mongodb.Server('localhost', 27017),
dbConnector = new mongodb.Db('example', mongoServer);
db_connector.open(function(err, db) {
if (err) throw new Error(err);
// here I can do my queries etc.
});
But how can I get access to the db object in the callback when I am in some route callback?
Currently the only idea I would have is wrapping the application into the callback:
var mongodb = require("mongodb"),
express = require("express"),
mongoServer = new mongodb.Server('localhost', 27017),
dbConnector = new mongodb.Db('example', mongoServer);
var app = new express();
db_connector.open(function(err, db) {
if (err) throw new Error(err);
app.get('/products', function(req, res, next) {
db.collection('products', function(err, collection) {
if (err) next(new Error(err));
collection.find({}, function(err, products) {
res.send(products);
});
});
});
});
But I do not think this is the way it should meant to be?
Isn't there the way to create a sync database connection call which I then can easily use through the whole application how it was by mongoose?
Regards bodo
Db.open opens the connection to mongodb and returns a reference to itself. See here for the sourcecode: https://github.com/mongodb/node-mongodb-native/blob/master/lib/mongodb/db.js#L245
All you want is to hold off on starting your express app listening on it's port and receiving requests until your connection to mongodb has been established.
So what you could do is this:
var mongodb = require("mongodb"),
express = require("express"),
mongoServer = new mongodb.Server('localhost', 27017),
dbConnector = new mongodb.Db('example', mongoServer),
db;
var app = new express();
app.get('/products', function(req, res, next) {
db.collection('products', function(err, collection) {
if (err) next(new Error(err));
collection.find({}, function(err, products) {
res.send(products);
});
});
});
db_connector.open(function(err, opendb) {
if (err) throw new Error(err);
db = opendb;
app.listen(3000);
});
What I'm not sure about though is whether this is a good idea. This solution doesn't allow you to recreate your connection if there has been a connection break or you restarted your mongodb process. So while the above might work, it might be a better idea to create a method that will wrap the creation of a connection to mongodb.