MongoDB not closing connections from NodeJS app - node.js

I have a node application that makes a call to mongoDB every 10 seconds, but looking at the output in my terminal, the connections just keep counting up and never seem to close:
My code to hit the ddb every 10 seconds:
const MongoClient = require("mongodb").MongoClient
setInterval(function(){
MongoClient.connect(uri, (err, client) => {
if (err){
console.log(err);
}
database = client.db(databaseName)
getData(function(data){
if(data.length > 0){
db_response = data;
params["fieldA"] = db_response[0]['fieldA'];
}
})
})
}, 10000)
function getData(callback){
var query = { fieldA: "foo" };
database.collection(CollectionName).find(query).toArray(function(err, result){
if (err){
throw err;
}
callback(result);
})
}
(The vars uri, CollectionName and databaseName are declared earlier) I guess what i need to do (and havent yet figured out) is to connect to the DB once when the server starts, and then run the getData() function on successful connection, does that mean the database variable needs to be a global var??

As you correctly identified you only need to create your db connection once. So rather than wrapping the the db connection creation with setInterval, wrap setInterval around the only function you want to repeat, in this case getData.
On your other question, the database variable doesn't need to global but you are right getData does need to use it. Therefore pass it as an argument along with your callback function.
If you want to close your connection use client.close(); inside MongoClient.connect
const MongoClient = require("mongodb").MongoClient
MongoClient.connect(uri, (err, client) => {
if (err){
console.log(err);
}
const database = client.db(databaseName);
setInterval(function(){
getData(database, function(data){
if(data.length > 0){
db_response = data;
params["fieldA"] = db_response[0]['fieldA'];
}
})
}, 10000)
})
function getData(db, callback){
var query = { fieldA: "foo" };
db.collection(CollectionName).find(query).toArray(function(err, result){
if (err){
throw err;
}
callback(result);
})
}

Related

How to continuously query a mongodb database efficiently

I'm new to Node and MongoDB and I have a seemingly simple request. I've managed to connect to my database, and use a query to get my desired results. Now, I want to have this query continue indefinitely, since the end goal for my project is to plot data real time.
I would have thought a simple 'while (true)' loop would suffice, but that doesn't seem to be the case.
const MongoClient = require('mongodb').MongoClient;
// Connection URL
const url = 'mongodb://<username>:<password>#ds157614.mlab.com:57614/flight_data';
// Use connect method to connect to the Server
MongoClient.connect(url, { useNewUrlParser: true }, function(err, db) {
if (err) throw err;
var dbo = db.db("flight_data").collection("data");
while(true)
{
dbo.find().sort({_id: 1}).limit(1).toArray(function(err, result) {
if (err) throw err;
console.log("Temperature: " + result[0].data.temperature);
});
}
db.close();
});
I have found that the while loop is indeed running, but for some reason, the query just doesn't happen when inside the while loop. If you remove the while loop, the code functions fine. I just want it to continually print the results of the query being repeated.
Querying a DB continuously is inefficient and resource wasting, instead use change streams. It watches collection for any changes and will make the db call then only. Works only for Mongo 3.6+.
const MongoClient = require("mongodb").MongoClient;
// Connection URL
const url =
"mongodb://<username>:<password>#ds157614.mlab.com:57614/flight_data";
// Use connect method to connect to the Server
MongoClient.connect(url, { useNewUrlParser: true }, function(err, db) {
if (err) throw err;
const collection = db.collection("data");
const changeStream = collection.watch();
changeStream.on("change", next => {
// process next document
collection
.find()
.sort({ _id: 1 })
.limit(1)
.toArray(function(err, result) {
if (err) throw err;
console.log("Temperature: " + result[0].data.temperature);
});
});
db.close();
});

Using NodeJS promise to query MongoDB

I am building a chatbot using WATSON API which sends artist data give users' input. I am trying to use nodejs promise in order to query my DB and print out the data, since DB accessing is asynchronous.
So the artpromise function is a function which takes in the artist's name and query the db to save the result in the 'result' variable. Then I am trying to print out the result (in chatbot i actually print out the result to the user).
However I am not getting the result I want and keep getting a syntax error. Any help would be appreciated.
let arttistinfo;
function artpromise (artist) {
return new Promise(function(resolve, reject) {
const MongoClient = require("mongodb").MongoClient;
const url = 'mongodb://majac.co.kr:27017/artbot';
MongoClient.connect(url, function(err, db) {
if (err) throw err;
var dbo = db.db("artbot");
var query = {name: artist};
artistinfo = dbo.collection("artistdb").find(query)
.toArray(function(err, result) {
if (err) throw reject(err);
resolve(result);
});
db.close();
}
});
)};
let artist = "Jan Tarasin";
artpormise.then(function(artist) {
console.log(result);
});
I'd rewrite like so, I can see there were a small number of issues with your code, but this works for me now:
function artpromise (artist) {
return new Promise(function(resolve, reject) {
const MongoClient = require("mongodb").MongoClient;
const url = 'mongodb://majac.co.kr:27017/artbot';
MongoClient.connect(url, function(err, db) {
if (err) throw err;
var dbo = db.db("artbot");
var query = {name: artist};
artistinfo = dbo.collection("artistdb").find(query)
.toArray(function(err, result) {
if (err) throw reject(err);
resolve(result);
});
db.close();
});
});
};
let artist = "Jan Tarasin";
artpromise(artist).then(function(result) {
console.log(result);
});
I get the result below:
[{
_id: 5abdbc18423795deaaff0d8e,
nationality: 'Polish',
art_link: 'https: //media.mutualart.com/Images/2016_06/29/20/203606422/0532d043-71f6-47bc-945e-aeededd2d483_570.Jpeg',
years: '1926',
name: 'JanTarasin',
art_title: '"Falujace watki I",
2003r.'
}]
MongoDB Node driver is natively supporting promises from v3 on. So you may greatly simplify your code by using them.
Here is how i would approach to your problem;
function artpromise (artist) {
const MongoClient = require("mongodb").MongoClient;
return MongoClient.connect('mongodb://majac.co.kr:27017') // connect to mongo server
.then(mc => mc.db('artbot') // get mongoClient object and connect to artbot db
.collection('artistdb') // connect to the artistdb collection
.find({name: artist}) // perform your query
.toArray() // convert the results into an array
.then(as => (mc.close(), as))) // close db and return array from query result
.catch(e => console.log(e)); // catch errors
}
let artist = "Jan Tarasin";
artpromise(artist).then(as => as.forEach(a => console.log(a)));
[nodemon] starting `node maeror.js`
{ _id: 5abdbc18423795deaaff0d8e,
nationality: 'Polish',
art_link: 'https://media.mutualart.com/Images/2016_06/29/20/203606422/0532d043-71f6-47bc-945e-aeededd2d483_570.Jpeg',
years: '1926',
name: 'Jan Tarasin',
art_title: ' "Falujące wątki I", 2003 r. ' }
[nodemon] clean exit - waiting for changes before restart
It might be useful to remind that cursor.toArray() returns a promise as it has to iterate all the query results at once before consturcting the results array. Sometimes this operation might be time consuming yielding delayed server response. So you may instead use the cursor.forEach() method to process the documents returned from the query one by one like a stream. Which means processing the first document and then iterating to the next one. Here is another example to show how it might be implemented.
function artpromise (artist) {
const MongoClient = require("mongodb").MongoClient;
return MongoClient.connect('mongodb://majac.co.kr:27017') // connect to mongo server
.then(function(mc){
var cursor = mc.db('artbot') // get mongoClient object and connect to artbot db
.collection('artistdb') // connect to the artistdb collection
.find({name: artist}); // get the cursor
return [mc, cursor]; // return mongoClient and cursor objects
});
}
let artist = "Italian";
artpromise(artist).then(function([mc,docs]){
docs.forEach(doc => console.log(doc), // process a document and then iterate to the next
() => mc.close()); // close db session when all documents are processed
})
.catch(e => console.log(e)); // catch errors
[nodemon] starting `node maeror_v2.js`
{ _id: 5abdbc18423795deaafeff13,
nationality: 'Dutch',
art_link: 'https://media.mutualart.com/Images/2012_04/15/13/132154856/ddf14e9d-85b1-4b5a-b621-00583e013879_570.Jpeg',
years: '1839 - 1902',
name: 'Frederick Hendrik Kaemmerer',
art_title: ' A Beach Stroll ' }
[nodemon] clean exit - waiting for changes before restart

Serverless - AWS Lamda with MongoDB Atlas Queries Not Running

I've created an AWS account and want to use MongoDB Atlas with AWS Lambda.
The only dependency I've downloaded is mongodb locally.
npm install mongodb
Driver based connection string given from mongoDB Atlas for Nodejs is
var uri = "mongodb+srv://kay:myRealPassword#cluster0.mongodb.net/test";
MongoClient.connect(uri, function(err, client) {
const collection = client.db("test").collection("devices");
// perform actions on the collection object
client.close();
});
I think the connection is successful, because err parameter is NULL.
But I cannot figure out how to create collection, how to find results, how to insert documents.
I've tried this code
module.exports.hello = (event, context, callback) => {
var MongoClient = require('mongodb').MongoClient;
var uri = "mongodb+srv://kay:myRealPassword#cluster0.mongodb.net/test";
MongoClient.connect(uri, function(err, client) {
const collection = client.db("test").collection("devices");
collection.insert( { "msg" : "My First Document" } );
var results = client.db("test").collection("devices").find();
console.log(results);
client.close();
callback(null, { message: 'Go Serverless v1.0! Your function executed successfully!', event });
});
};
but it returns (in Windows console) a huge Object in JSON format, its like a configuration data (not a query result)
enter image description here
I'm executing this code locally by
sls invoke local --function hello
The general idea is to check if there is an error in the connection, the insert, and so on. Take at look at this error checking:
if (error) return 1;
There are more sophisticated methods, but for your case this should do the work.
This is a example of how it show look your script:
MongoClient.connect(uri, (error, client) => {
if (error) return 1; // Checking the connection
console.log('Connection Successful');
var db = client.db('mydb'); // Your DB
let newDocument = { "msg" : "My First Document" }; // Your document
db.collection('mycollection').insert(newDocument, (error, results) => { // Your collection
if (error) return 1; // Checking the insert
console.log('Insert Successful');
})
db.collection('mycollection')
.find({})
.toArray((error, accounts) => {
if (error) return 1; // Checking the find
console.log('Find Successful');
console.log(accounts);
return 0;
})
})
And you should have an output like this:
Connection Successful
Insert Successful
Find Successful
[ { _id: 5a857dd2c940040d85cbe5f2, msg: 'My First Document' } ]
If your output is not like this, well the missing log would point the place where you have your error.

NodeJS Mongoose collection.remove not working

Trying to drop all docs in all collections before unit tests run...
var collections = mongoose.connection.collections;
async.eachSeries(_.keys(collections), function(key, cb){
collections[key].remove(function(){
//Never gets here, doesn't drop the collection and doesn't error.
cb();
});
}
But the callback in remove() never gets fired.
I've logged out collections[key] and it does resolve to a collection.
No errors, but it times out as it never runs the callback.
Ive tried looping the Models as well and calling that remove but its the same issue.
What am I doing wrong here?? Any logs I could look at?
You could try using the drop method:
var async = require("async"),
_ = require("underscore"),
collections = _.keys(mongoose.connection.collections);
async.forEach(collections, function(key, done) {
var collection = mongoose.connection.collections[key]
collection.drop(function(err) {
if (err && err.message != "ns not found") {
done(err);
} else {
done(null);
}
})
}, callback);
EDIT: Try the following:
var collections = mongoose.connection.collections;
async.eachSeries(_.keys(collections), function(key, cb){
mongoose.connection.db.collection(key, function(err, col) {
col.remove({}, function(){
cb();
});
})
}
Unrelated issue, it wasn't connecting to the DB in the test environment. But no errors were reported, and it had a valid list of collections due to mongoose models.
I added the following to the test set up to log out the errors and other info to help find these issues in the future...
mongoose.connection.on('connected', function () {
console.log('Mongoose default connection open to ' + config.db.url);
});
mongoose.connection.on('error',function (err) {
console.log('Mongoose default connection error: ' + err);
});
mongoose.connection.on('disconnected', function () {
console.log('Mongoose default connection disconnected');
});

My mongoose test fails unless I repeatedly check the data I've saved in the database

I am having trouble writing tests for my objects using mocha. The code appears to work in practice but during my tests, database updates are happening later than expected. During test setup I insert a few records into a collection. In the teardown I clear the collection. When I search for a known record I get zero results unless I recursively invoke the callback function (as shown in the code below).
Everything is written asynchronously. The setup function returns all the records. Somehow it seems that the data is not refreshed quickly enough. Can anyone advise if I am approaching this correctly with the recursive loop?
var myclass = require('myclass')
var mongoose = require('mongoose');
var should = require('should');
mongoose.connect('mongodb://localhost/myDbTests');
mongoose.connection.on('error', console.error.bind(console, 'connection error:'));
describe('Test my collection.', function () {
beforeEach('load dummy data into the database', function (done) {
myclass.load_data(dummyData, function (count) {
count.should.be.greaterThan(5);
done();
});
});
afterEach('clear the database', function (done) {
myclass.model.remove({}, function() {
done();
});
});
it('check that a known record exists database', function (done) {
var keep_checking = function (td) {
if (!td) {
myclass.get_record('MYRECORD', keep_checking);
} else {
td.should.have.property('category', 'someCategory');
done();
}
}
keep_checking(0);
});
});
My load_data is:
var _load_data = function (data, callback) {
data.forEach(function (d) {
var rec = new _model(d);
rec.save(function(err, res) {
if (err) return console.error(err);
});
});
callback(data.length);
};
You should wait until the database connection is open to run your tests.
I achieve that in my tests with the before hook which runs before any test (and before beforeEach as well):
before(function(done) {
mongoose.connection.once('open', done);
}
That will prevent anything from being run before the database connection is open.
I was not loading the data correctly. Mongoose does not allow multiple record insertion so I had used a synchronous .forEach loop to save each object. A better way to do, and hence my solution, is the following:
var _load_data = function (data, callback) {
var total = data.length,
count = 0;
function saveAll() {
var doc = data[count];
var rec = new _model(doc);
rec.save(function(err, res) {
if (err) {
throw err;
}
if (count !== total) {
count += 1;
saveAll();
} else {
callback(count);
}
});
}
saveAll();
};

Resources