How to efficiently close the db connection (mongodb and node) - node.js

I have a small program that reads each of the record and update each of the record. Given the async nature of node and callback. what is the efficient and the correct way to close the db connection?
Sample Program:
var MongoClient = require('mongodb').MongoClient;
var updateCount = 0;
MongoClient.connect('mongodb://localhost:27017/school', function(err, db) {
if(err) throw err;
var query = { };
// get all the students in the database
var cursor = db.collection('students').find(query);
cursor.each(function(err, doc) {
if(err) throw err;
if(doc == null) {
return;
}
// filter out only the homework scores
var homeworksOnly = doc.scores.filter(function(scores){
if (scores.type === "homework") return true;
return false;
})
// filter out the non homework scores
var notHomeWorks = doc.scores.filter(function(scores){
if (scores.type !== "homework") return true;
return false;
})
// sort the homework score to remove the min score from the list.
homeworksOnly.sort(function(a,b){
if (a.score > b.score) return 1;
if (b.score > a.score) return -1;
return 0;
});
console.log("Before removing the min score"+doc._id);
console.dir(homeworksOnly);
console.log("After removing the min score"+doc._id);
homeworksOnly.splice(0,1);
console.dir(homeworksOnly);
console.log("Merge the homework with other scores"+doc._id);
var newScores = homeworksOnly.concat(notHomeWorks);
console.dir(newScores);
console.log("*****");
// Now update the database for this student with the new scores
var search = {"_id":doc._id};
var operator = { '$set' : { 'scores' : newScores } };
db.collection('students').update(search, operator, function(err, updated) {
if(err) throw err;
updateCount++;
console.dir("Successfully updated " + updated + " document! count: "+updateCount);
});
});
});
Now the program works but I need to hit the Ctrl+C to terminate the program. Is there a way to know that all the callbacks have completed so that the program can be terminated?

There are better libaries you can integrate with nodejs to handle the callback flow better, but simply working with the basic driver as a dependency, all you need is the basic node stream interface which is already built in to the cursor.
This allows .pause() and .resume()for flow control when processing, and an "end" event when the cursor stream is complete:
var MongoClient = require('mongodb').MongoClient;
var updateCount = 0;
MongoClient.connect('mongodb://localhost:27017/school', function(err, db) {
if(err) throw err;
var query = { };
// get all the students in the database
var cursor = db.collection('students').find(query);
// called on errors
cursor.on("error",function(err) {
throw err;
});
// called on stream complete
cursor.on("end",function() {
db.close();
});
// process each document in the stream
cursor.on("data",function(data) {
cursor.pause(); // stops the cursor stream while processing
// filter out only the homework scores
var homeworksOnly = doc.scores.filter(function(scores){
if (scores.type === "homework") return true;
return false;
})
// filter out the non homework scores
var notHomeWorks = doc.scores.filter(function(scores){
if (scores.type !== "homework") return true;
return false;
})
// sort the homework score to remove the min score from the list.
homeworksOnly.sort(function(a,b){
if (a.score > b.score) return 1;
if (b.score > a.score) return -1;
return 0;
});
console.log("Before removing the min score"+doc._id);
console.dir(homeworksOnly);
console.log("After removing the min score"+doc._id);
homeworksOnly.splice(0,1);
console.dir(homeworksOnly);
console.log("Merge the homework with other scores"+doc._id);
var newScores = homeworksOnly.concat(notHomeWorks);
console.dir(newScores);
console.log("*****");
// Now update the database for this student with the new scores
var search = {"_id":doc._id};
var operator = { '$set' : { 'scores' : newScores } };
db.collection('students').update(search, operator, function(err, updated) {
if(err) throw err;
updateCount++;
console.dir("Successfully updated " + updated + " document! count: "+updateCount);
cursor.resume(); // restarts the stream processing now we are done
});
});
});

After the update statement is done use
db.collection('students').update(search, operator, function(err, updated) {
if(err) throw err;
updateCount++;
console.dir("Successfully updated " + updated + " document! count: "+updateCount);
});
db.close();

Related

nodejs get sqlite3 query result using promise or wait

This is my first personal project in Nodejs. I'm trying to get in live soon.
I have a Nodejs server that uses sqlite3. There are only 3000 rows with word, transform and a precalculated value each in a column of the table, which is already populated.
I need to just lookup the word in the DB to be sure it is valid.
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
db.get("SELECT * FROM tab WHERE w = ?", word, function(err, row) {
if(err) { console.log("Lookup:",word,", Error => ",err); return false; }
return true;
});
The problem is that the caller of this code has a lot of context and need the operation to wait. So, I tried this
function dbLookup(db, w) {
return function(cb) {
var rows = [];
db.exec('SELECT w FROM tab WHERE w = "'+w+'"')
.on('row', function(r) {
rows.push(r)
})
.on('result', function() {
cb(rows);
});
}
async.each([word], function(w) {
dbLookup(this.db, w);
}, function(err) {
if(err) {console.log("...ERROR..."); return false; }
else {console.log("...SUCCESS..."); return true; }
});
This doesn't solve the wait issue as the callback can fire at its own pace.
I read that promise using something like bluebird can solve my problem
but now I'm not able to get the value/result of the query out:
I've been pulling my hair for so long. Please help me either get the async working or get the result back from the promise approach.
var async = require('async');
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
function check(word, callback) {
db.get("SELECT count(1) cnt FROM tab WHERE w = ?", word, callback)
}
async.map(words, check, function(err, results) {
if (err)
return console.log('Query error')
var all_checked = results.filter(function(r) {
return r.cnt > 0
});
...
});
Or
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
db.all("SELECT distinct w FROM tab", function(err, rows) {
var all_checked = words.filter(function (w) {
return rows.indexOf(w) != -1;
})
...
})

MongoDB find function running twice in node.js

I'm following this link for finding data in mongoDB using node.js
My code is:
var counter = 0;
var findMongo = function(db, callback) {
var cursor =db.collection('new').find( { "_id": ObjectId("56da6fd166efee0350399c21") } );
//var cursor =db.collection('new').find();
cursor.each(function(err, doc) {
counter = counter + 1;
console.log(counter);
assert.equal(err, null);
if (doc != null) {
//console.dir(doc);
//console.log(doc);
} else {
console.log("in else,not found");
callback();
}
});
};
MongoClient.connect(url, function(err, db) {
assert.equal(null, err);
findMongo(db, function() {
db.close();
});
});
Since I'm searching the DB with _id, findMongo should only run once.
I'm getting following result:
counter 1
counter 2
in else,not found
Why is the findMongo function called twice?
Two things to be noticed:
1 - You are using counter = counter + 1; twice, its just creating confusion.
2 - You should use findOne instead find, it makes sense and is good approach because you are interested in finding one-record only whereas there is no harm in using later one.
Here is how to use db.collection.findOne()

MongoDB connection ends prematurely

var MongoClient = require("mongodb").MongoClient;
MongoClient.connect("mongodb://localhost:27017/test",function(err,db){
if(err) throw err;
var query = {};
var projection = {State : 1,Temperature : 1,_id : 1};
var options = {
sort : [["State","asc"],["Temperature","desc"]]
};
var state = "";
var temp = 0;
db.collection("weather").find(query,projection,options).each(function(err,doc){
if(err) throw err;
if(doc){
// Check if we have found a new state
if(state !== doc.State){
state = doc.State;
doc["month_high"] = true;
db.collection("weather").save(doc,function(err,saved){
if(err) throw err;
});
}
}else{
return db.close();
}
});
});
This is my code where I have to update certain documents that meet a criteria. Instead of storing all the documents that need to be updated, and then updating them later, I am updating them as they come.
However, I get a MongoError: server localhost:27017 sockets closed. I know this is because I am trying to use a connection that has already been closed.
How do I properly sequence the closing of the database and updating of the documents?
It looks like your last db.save() is still being processed when you call db.close()
Check if you are processing the last item and then call db.close() in your db.save() callback.

synchronous loop in nodejs

//Importing: postges DB connection
var pg = require('pg');
var conString = "postgres://readxxx:p#ssword#vmwoxxx-tst:8888/worxxx";
var prvsiteid = '';
var cursiteid = '';
var qurystring = '';
pg.connect(conString, function(err, client, done) {
if (err) {
return console.error('error fetching client from pool', err);
return;
}
client.query("select site_id,created_at,started_at,completed_at,notes,finish_code from _background_tasks where finish_code > 0 and site_id > 0 and abs(extract(Epoch from (now()::timestamp without time zone - completed_at)))/60 <= 4600 order by site_id asc", function(err, result1) {
done();
if (err) {
return console.error('error running query', err);
return;
}
for (var i = 0; i < result1.rowCount; i++) {
cursiteid = result1.rows[i].site_id;
if (prvsiteid != cursiteid) {
prvsiteid = cursiteid;
qurystring = "select trim(su.name) as name, case When trim(su.email) is null then su.name || '#netapp.com' when trim(su.email) > '' then su.name || '#netapp.com' else su.name end uemail,su.friendly_name as frdname from system_users su where su.state = 'active' and su.id in (select distinct(system_user_id) from users u where u.site_id = " + cursiteid + "and u.admin_level >= 5)"
client.query(qurystring, function(err, result2) {
done();
if (err) {
return console.error('error running query', err);
return;
}
for (var j = 0; j < result2.rowCount; j++) {
console.log(cursiteid, result2.rows[j].name, result2.rows[j].uemail, result2.rows[j].frdname);
}
});
}
console.log(result1.rows[i].site_id, result1.rows[i].created_at, result1.rows[i].started_at, result1.rows[i].completed_at);
}
});
});
I know NodeJS programs are asynchronous but this scenario I intend it to be synchronous.
for loop(outer) --> for loop(inner) when outer forloop changes with new site id i want to send email to all the emailids from inner loop and also the resultant rows of the each site of outer loop has to printed.
If the library you are using doesn't support synchronous operations, you can't use it synchronously without abominations (You do not want abominations in your code. No matter how much the boss is pressing).
You can fairly simply perform loops and other actions on asynchronous operations by using Promises, specifically, bluebird which supports the ability to take callback-based asynchronous functions and transform them into Promise-ready functions.

mongodb doesn't save all data

I need to insert some matrix in mongodb,so I wrote simple following code
var MongoClient = require('mongodb').MongoClient;
var matrisMaker = function(d1,d2){
var result = new Array();
for (var i = 0;i < d1;i++){
result.push(new Array());
for (var k = 0;k < d2;k++){
result[i].push(Math.round(Math.random() * 1000000000000));
}
}
return result;
};
MongoClient.connect('mongodb://127.0.0.1:27017/test', function(err, db) {
if(err) throw err;
var collection = db.collection('matris');
for (var counter = 0;counter < 10000000;counter++){
var insertObject = {
'matrisA':matrisMaker(20,20),
'matrisB':matrisMaker(20,20),
'resultA':new Object(),
'resultB':new Object()
};
collection.insert(insertObject, function(err, docs) {
if (err)
throw err;
});
delete insertObject;
if ((counter % 1000) == 0)
console.log(counter);
}
db.close();
})
when I see log it printed that too many records inserted,like 50,000,but when I use mongodb to count amount of records,it display less,something near 1,000 records.
>use test;
>db.matris.count();
where is the problem?
Your asynchronous code is flawed and your db.close() line executes before your asynchronous insert commands have all completed. You need to control the flow of your program to A) not have a million concurrent database inserts happening/queued and B) wait until they have all been processed by mongo before closing the connection. Consider a helper library such as async.forEach to help with this if you don't want to code it yourself.

Resources