Node.js + MongoDB update operation confirmation - node.js

In Node.js, I have (briefly) this script:
var http = require('http');
var XmlStream = require('xml-stream');
var mongo = require('mongodb');
var tims = { ... };
var db = new mongo.Db('tims', new mongo.Server("127.0.0.1", 27017, {}), {w: 1});
db.open(function(e, db) {
var req = http.get({
host: tims.uri,
path: '/xml/'+tims.database+tims.services.database
}).on('response', function(res) {
res.setEncoding('utf8');
cProjects = db.collection("projects");
var xml = new XmlStream(res);
xml.on('updateElement: Tims ProjectID', function(project) {
// console.log(project.$text+' - '+project.$.title);
cProjects.update({project_id: project.$text}, {project_id: project.$text, title: project.$.title}, {upsert:true}, function(err, result) {
console.log('result: '+result);
});
});
xml.on('end', function(data) {
db.close();
});
});
});
I am using a Node.js package called xml-stream that pieces together response chunks from Node to get valid XML before processing. My problem: if I leave out
xml.on('end', function(data) {
db.close();
});
my connection never closes and the console just hangs. The upside is that console.log('result: '+result); writes to the console and I can see that my data was committed successfully. So if I leave in the end event and close the DB once all XML has been processed, the Node instance terminates before console.log('result: '+result) is written.
I am a newbie to both MongoDB and Node.js, so I was curious what the best practice is here for confirmation, or perhaps a simple pointing out of what I'm doing wrong.
Thanks for the help.

Looks like the 'end' event is occurring before all the update callbacks have completed. So you need to rework your code a bit to keep track of the number of updates still pending and only call db.close() once both the 'end' event has fired and all the pending updates have completed.
So something like this:
db.open(function(e, db) {
var req = http.get({
host: tims.uri,
path: '/xml/'+tims.database+tims.services.database
}).on('response', function(res) {
res.setEncoding('utf8');
cProjects = db.collection("projects");
var xml = new XmlStream(res);
var end = false;
var pending = 0;
xml.on('updateElement: Tims ProjectID', function(project) {
// console.log(project.$text+' - '+project.$.title);
++pending;
cProjects.update({project_id: project.$text}, {project_id: project.$text, title: project.$.title}, {upsert:true}, function(err, result) {
console.log('result: '+result);
if (--pending === 0 && end) {
db.close();
}
});
});
xml.on('end', function(data) {
end = true;
});
});
});

Related

RabbitMQ request, always timesout

I have a weird problem where my callback is never published and the message goes to timeout, even though the method runs in the queue. This happens in some specific queues and after it happens once, i cannot make any other requests from client which even previously worked, they all timeout. Have to restart the client and sever to make it working again.
This is the code, where its happening, and i cant seem to understand whats wrong.
Server.js file where i am creating the queues. I have several such queues, this is one of them.
var amqp = require('amqp');
var util = require('util');
var cnn = amqp.createConnection({host:'127.0.0.1'});
var getCart = require('./services/getCart');
cnn.on('ready', function() {
cnn.queue('getCart_queue', function(q){
q.subscribe(function(message, headers, deliveryInfo, m){
// util.log(util.format( deliveryInfo.routingKey, message));
// util.log("Message: "+JSON.stringify(message));
// util.log("DeliveryInfo: "+JSON.stringify(deliveryInfo));
getCart.handle_request(message, function(err,res){
cnn.publish(m.replyTo, res, {
contentType:'application/json',
contentEncoding:'utf-8',
correlationId:m.correlationId
});
});
});
});
});
Here, the handle request function is completed successfully, but the callback never goes through and its always timeout on the other end
var cart = require('../models/cart');
function handle_request(msg, callback) {
var user_id = msg.id;
cart
.find({id:user_id})
.populate('users ads')
.exec(function(err, results){
// This works, just the callback doesnt
if(!err){
console.log(results);
callback(null, results);
} else {
console.log(err);
callback(err, null);
}
});
}
exports.handle_request = handle_request;
this is how i am calling the request
var msg_payload = {"id":id};
mq_client.make_request('getCart_queue', msg_payload, function(err, results){
console.log(results); // never prints
//stuff that is never reached
});
These are my rpc files, i dont think there should be anything wrong with these, as some other queues work fine.
And this is the error shown on client
GET /getCart - - ms - -
Error: timeout 6ee0bd2a4b2ba1d8286e068b0f674d8f
at Timeout.<anonymous> (E:\Ebay_client\rpc\amqprpc.js:32:18)
at Timeout.ontimeout [as _onTimeout] (timers.js:341:34)
at tryOnTimeout (timers.js:232:11)
at Timer.listOnTimeout (timers.js:202:5)
Hope the information is not vague, if you need more, please let me know. Thanks!
I Think the error is in this file, because i tried debugging and from the rabbitmq server, the callback is being called and it has the correlation id as well as the reply to variable, so the request is not getting picked up here.
var amqp = require('amqp')
, crypto = require('crypto');
var TIMEOUT=8000;
var CONTENT_TYPE='application/json';
var CONTENT_ENCODING='utf-8';
var self;
exports = module.exports = AmqpRpc;
function AmqpRpc(connection){
self = this;
this.connection = connection;
this.requests = {};
this.response_queue = false;
}
AmqpRpc.prototype.makeRequest = function(queue_name, content, callback){
self = this;
var correlationId = crypto.randomBytes(16).toString('hex');
var tId = setTimeout(function(corr_id){
callback(new Error("timeout " + corr_id));
delete self.requests[corr_id];
}, TIMEOUT, correlationId);
var entry = {
callback:callback,
timeout: tId
};
self.requests[correlationId]=entry;
self.setupResponseQueue(function(){
self.connection.publish(queue_name, content, {
correlationId:correlationId,
contentType:CONTENT_TYPE,
contentEncoding:CONTENT_ENCODING,
replyTo:self.response_queue});
});
};
AmqpRpc.prototype.setupResponseQueue = function(next){
if(this.response_queue) return next();
self = this;
self.connection.queue('', {exclusive:true}, function(q){
self.response_queue = q.name;
q.subscribe(function(message, headers, deliveryInfo, m){
var correlationId = m.correlationId;
if(correlationId in self.requests){
var entry = self.requests[correlationId];
clearTimeout(entry.timeout);
delete self.requests[correlationId];
entry.callback(null, message);
}
});
return next();
});
};
This is the code for your make_request() in client.js file:
var amqp = require('amqp');
var connection = amqp.createConnection({host:'127.0.0.1'});
var rpc = new (require('./amqprpc'))(connection);
function make_request(queue_name, msg_payload, callback){
rpc.makeRequest(queue_name, msg_payload, function(err, response){
if(err)
console.error(err);
else{
console.log("response", response);
callback(null, response);
}
});
}
exports.make_request = make_request;
Look at what happens when you have an err on rpc.makeRequest():
rpc.makeRequest(queue_name, msg_payload, function(err, response){
if(err)
console.error(err);
//
//HERE: should be a callback call here.
//
else{
console.log("response", response);
callback(null, response);
}
});
This could be why you are getting a timeout. I hope it helps.
There wasn't a problem with rabbitMQ but with my queries in the handle request and after responding to the request.
For others coming with this problem, check and double check every statement, as the error will not show in the console, but will only show a timeout

Proper way to use pg-postgres in node.js?

I am new to pg-postgres for node.js. I am trying to figure out the proper way to make queries. Right now I have this
var client = new pg.Client(connectionString);
client.connect();
And then in each of my http request routes, I have this kind of code:
var query = client.query(sql);
query.on('row', function(row, result) {
result.addRow(row);
});
query.on('end', function(data) {
if (data.rows[0].count === '1') {
return callback();
}
return failedCallback(req, res);
});
Is this the way to do it? Or should I do this in each http route handler
pg.connect(conString, function(err, client) {
// Use the client to do things here
client.end();
});
Also do I need to end the client in each http route handler in this way?
Matter of choice:
It is a matter of choice. The first allows you to reuse declared client in other functions, while the second client is inside your pg.connect. I don't think you can say one is proper and the other is not. I can rephrase the question as "what is better: c.query(sql, function (err, result) { or query.on('end', function(data) {".
Regarding client.end():
If you end connection in each http route handler, you won't be able to reuse client, unless you client.connect(); again. It does not mean though that you don't have to close it at all. I believe bonding client termination to response.send() is reasonable. If you don't close connections to postgres you will reach max_connections quite fast. Again - you will probably use connections pooling instead of connecting to pg directly, but it does not mean you should generate new and new connections endlessly.
Example of both:
//https://github.com/brianc/node-postgres
var pg = require('pg');
var conString = 'postgres://n#10.1.10.199/b';
var sql = 'SELECT substr(extract(epoch from now())::text,10,4) sec, $1::int client,pg_backend_pid() pid,count(*) from pg_stat_activity where usename = current_user group by client';
var client = new pg.Client(conString);
client.connect();
var query = client.query(sql,[1]);
query.on('end', function(data) {
console.log(data.rows);
});
var query = client.query(sql,[1]);
query.on('end', function(data) {
console.log(data.rows);
//client.end();
});
var to = setTimeout( //we start new client with small timeout to see one session with previous client
function() {
pg.connect(conString, function(err, c) {
// execute a query on our database
c.query('select pg_sleep(0.6);', function (err, result) {
});
c.query(sql,[2], function (err, result) {
if (err) throw err;
console.log(result.rows);
});
c.query(sql,[4], function (err, result) {
console.log(result.rows);
});
//c.end();
to;
});
}
, 500
);
var to = setTimeout(
function() {
var query = client.query(sql,[3]);
query.on('end', function(data) {
console.log(data.rows);
//client.end();
});
}
, 1000
);
Generates:
...js>node 1.js
[ { sec: '2.31', client: 1, pid: 23327, count: '1' } ]
[ { sec: '2.32', client: 1, pid: 23327, count: '1' } ]
[ { sec: '3.29', client: 3, pid: 23327, count: '2' } ]
[ { sec: '3.41', client: 2, pid: 23328, count: '2' } ]
[ { sec: '3.42', client: 4, pid: 23328, count: '2' } ]
As you can see, client 1 and client 3 share same pid, although called in different functions, because global var client was declared. Same for 2 and 4, called async in one pg.connect share one pid.
Now should you put your code into c.query(sql, function (err, result) { or query.on('end', function(data) { depends on what you want to achieve.

On moving to TDD with node and mongo

I want to use mocha to add to my program feature by feature, test by test.
var assert = require('assert');
var mongoskin = require('mongoskin');
describe('basic database tests', function(){
before(function(){
});
it('should have 3 users', function(done){
var db = mongoskin.db('mongodb://localhost:27017/stuffTest', {safe:true});
db.collection('users').find().toArray(function (err,result){
console.log(result.length);
assert.equal(result.length,3);
});
});
});
It doesn't work. I get an error no matter where I put stuff in the test. With this arrangement I get Error: timeout of 2000ms exceeded
This is the code to set up the database. My old way of development was to litter my code with console.logs and such. This code uses console.logs to let me know if the collection is empty and then if so was it filled with 3 records.
var mongoskin = require('mongoskin')
var db = mongoskin.db('mongodb://localhost:27017/stuffTest', {safe:true})
db.collection('users').find().toArray(function (err,result){
console.log(result.length)
})
db.collection('users', {strict:true}, function(err, collection) {
if (err) {
console.log("The 'users' collection doesn't exist. Creating it with sample data...");
populateDB(users);
}
});
var populateDB = function(huh) {
console.log("Populating database...");
var name= huh.name;
var coll= huh.items;
db.collection(name, function(err, collection) {
collection.insert(coll, {safe:true}, function(err, result) {
console.log(result.length);
});
});
};
var users = [];
users.name = 'users';
users.items= [
{name: 'tim', email: 'mckenna.tim#gmail.com', lists:[]},
{name: 'peri', email: 'perimckenna#gmail.com', lists:[]},
{name: 'tim2', email: 'mckt_jp#yahoo.com', lists:[]}
];
How would I write this test? This code plus package.json and dropDb.js is here: https://github.com/mckennatim/tdd
You're not calling done. If you don't call done in an asynchronous test you are guaranteed to get a timeout. Modify the test to call done at the end of your callback. Like this:
it('should have 3 users', function(done){
var db = mongoskin.db('mongodb://localhost:27017/stuffTest', {safe:true});
db.collection('users').find().toArray(function (err,result){
console.log(result.length);
assert.equal(result.length,3);
done();
});
});

Mongodb (mongolab) Node.js finding nothing on a collection

I'm a big Node.js and Mongo newbie, so please be gentle.
So here's my Node.js app:
var mongo = require('mongodb');
var Server = mongo.Server;
var Db = mongo.Db;
var server = new Server('hostname.mongolab.com', 666, {auto_reconnect : true}, {w:0, native_parser: false});
var db = new Db('dbName', server, {safe:true});
db.open(function(err, client) {
if(err) { return console.dir(err); }
client.authenticate('mongolabUser', 'mongolabUserPassword', function(authErr, success) {
if(authErr) { return console.dir(authErr); }
var stream = client.collection('myCollection').find({}).stream();
stream.on('data', function(item) {console.log("Do something with item"); });
stream.on('end', function() {console.log("Empty!");});
});
db.close();
});
Through prodigious use of debugger statements, I've come to the conclusion that the client.authenticate doesn't seem to be run. It looks like it's about to execute that line, but then just leapfrogs over it and goes straight to db.close().
But that's just the first of my problems. At some point prior, I was able to connect in to the database and authenticate, but my user was no retrieving anything in the find({}) command. I tried all sorts of ways, and streams are my latest attempt before deciding to give up on it for now.
Mongolab seems to be on v2.0.7, my mongo installation is v2.2.1. When I use the command line tool to log in as mongolabUser and execute a command like db.myCollection.find(), I get everything in my collection, so it can't be an issue with permissions.
Any advice/suggestions?
client.authenticate() is asynchronous, so the line that calls it starts the authentication, but doesn't wait for the server to respond before moving on to executing the next line, db.close(). So by the time the server responds the connection has been closed by the client.
Does moving the db.close() inside the event handler for stream.end help?
var mongo = require('mongodb');
var Server = mongo.Server;
var Db = mongo.Db;
var server = new Server('hostname.mongolab.com', 666, {auto_reconnect : true}, {w:0, native_parser: false});
var db = new Db('dbName', server, {safe:true});
db.open(function(err, client) {
if(err) { return console.dir(err); }
client.authenticate('mongolabUser', 'mongolabUserPassword', function(authErr, success) {
if(authErr) { return console.dir(authErr); }
var stream = client.collection('myCollection').find({}).stream();
stream.on('data', function(item) {console.log("Do something with item"); });
stream.on('end', function() {
console.log("Empty!");
db.close();
});
});
});

MongoDb driver in nodejs does not call open function callback

In the following code the function passed to the open function never runs, then the istruction console.log('open!') in the following code never runs:
var mongo = require("mongodb");
var Db = mongo.Db;
var connection = mongo.Connection;
var Server = mongo.Server;
var client = new Db('test', new Server("localhost", 27017, {}));
var test = function (err, collection) {
collection.insert({a:2}, function(err, docs) {
collection.count(function(err, count) {
test.assertEquals(1, count);
});
// Locate all the entries using find
collection.find().toArray(function(err, results) {
test.assertEquals(1, results.length);
test.assertTrue(results[0].a === 2);
// Let's close the db
client.close();
});
});
};
client.open(function(err, p_client) {
console.log('open!');
client.collection('test_insert', test);
});
From the log I see that the connection is accepted:
Sun March 11 16:52:01 [initandlisten] accepted connection from 127.0.0.1:61875 # 1
Mongodb great works from interactive shell.
can someone tell me any suggestion?
thank you!
Copy/paste this...it should work without any problems:
var client = new Db('test', new Server("localhost", 27017, {}), {});
client.open(function(err, client) {
console.log('open!');
}
Also, don't forget to authenticate after opening the connection.
var client = new Db('test', new Server("localhost", 27017, {}), {});
client.open(function(err, client) {
console.log('open!');
client.authenticate('admin', 'admin', function(err, result) {
// Authenticated
});
}

Resources