What will be the best way to reuse a CouchBase conneciton - node.js

I am using the following code for connecting the CouchBase
couchbase.connect(config.CouchBaseConnector, function (err, CouchBaseDB) {
if (err) {
throw (err)
}
CouchBaseDB.set(keyPush, docPush, function (err, meta) {
if (err) { console.log(err); }
});
}
But its creating multiple number of connection.
Can someone help me out to fix the issue. Basically I want to do something like connection pool and keep re-using.
I came across a doc from CouchBase regarding the same. But not able to figure it out how exactly it works and the steps to deploy the same on windows 7 64-bit version.
Update:
I think moxi-server is not released for Windows OS as of now.

The Couchbase Node SDK is a Connection Pool itself. It is responsible of managing connection to the cluster, be alerted about any change in the server topology (add/remove/failed nodes)
This is why most of the time you put your code in a global callback method and reuse the connection
var express = require('express'),
driver = require('couchbase'),
routes = require('./routes');
dbConfiguration = {
"hosts": ["my-couchbase-server:8091"],
"bucket": "bucket"
}
driver.connect(dbConfiguration, function(err, cb) {
if (err) {
throw (err)
}
// put your application logic here
});
If you want to use a global variable you need to wait for the callback and be sure that the connection is established before using it.

I found the following code is working for me.
Please anyone has better solution please post it, I always welcome that.
GLOBAL.CouchBaseDBConnection = undefined;
function OpenCouchBase(callback) {
if (CouchBaseDBConnection == undefined) {
couchbase.connect(config.CouchBaseConnector, function (err, couchbaseOpenCon) {
if (err)
return console.log("Failed to connect to the CouchBase");
else {
CouchBaseDBConnection = couchbaseOpenCon
callback(null, couchbaseOpenCon);
}
});
}
else { callback(null, CouchBaseDBConnection); }
};
module.exports.OpenPoolCouchBase = OpenCouchBase;

You can use generic resource pool module for Node: node-pool
It is generic pool, so you can adapt it for your needs.
EDIT:
Here is example code:
var poolModule = require('generic-pool');
var pool = poolModule.Pool({
name : 'couch',
create : function(callback) {
couchbase.connect(config.CouchBaseConnector, function (err, couchbaseOpenCon) {
if (err)
return console.log("Failed to connect to the CouchBase");
else {
CouchBaseDBConnection = couchbaseOpenCon
callback(null, couchbaseOpenCon);
}
});
},
destroy : function(client) { client.end(); },
max : 10,
// specifies how long a resource can stay idle in pool before being removed
idleTimeoutMillis : 30000,
// if true, logs via console.log - can also be a function
log : true
});
// acquire connection - callback function is called once a resource becomes available
pool.acquire(function(err, client) {
if (err) {
// handle error - this is generally the err from your
// factory.create function
}
else {
console.log("do whatever you want with the client ...");
pool.release(client);
});
}
});

Related

What is the 'callback' parameter in this common version of the mongodb.MongoClient.connect script?

NodeJS version: v10.16.0
MongoDB version: 3.3.0
I am comfortable using 4 different versions of mongoDB connect scripts. However on more than one occasion I've come across a specific type of connect script that I don't understand. Specifically I don't understand what the callback parameter is or how it is being used/called.
This is one version of this connect script that uses the callback parameter. How is this being used? Why is it necessary?
function startDb(callback) {
mongodb.MongoClient.connect(process.env.MONGODB_URI, function (err, database) {
if (err) {
console.log(err);
callback(err);
} else {
let db = database;
console.log("Database connection ready");
callback();
}
});
}
The callback function is being passed as the argument of startDb(). You can pass a function reference as as argument since everything is an object.
function sayHello() {
console.log('Hello')
}
function indirectFunction(callback) {
callback() // this will call the sayHello() function because it was passed as the argument
}
indirectFunction(sayHello) // logs 'Hello'
This is how it would be applied in the example you gave
function sayHello(response) {
console.log(response) // will either log 'Hello' or the value of err
}
function startDb(callback) {
mongodb.MongoClient.connect(process.env.MONGODB_URI, function (err, database) {
if (err) {
callback(err);
} else {
let db = database;
console.log("Database connection ready");
callback('Hello');
}
});
}
startDb(sayHello)

Redshift data access from node jdbc throwing errors

I am getting below error when I try to data from redshift with the below mentioned code: I feel that it is due to jdbc module not installed correctly.Could
/*
* GET users listing.
*/
var jdbc =require('jdbc');
var express = require('express');
var config = {
libpath: '../RedshiftJDBC41-1.1.6.1006.jar',
drivername: 'com.amazon.redshift.jdbc41.Driver',
url: 'jdbc:redshift://exampleCluster.abcdkcmf0ug1.us-west-2.redshift.amazonaws.com:5439/xyz',
properties: [
['user', 'XXXXX'],
['password', 'XXXXXX']
]
};
console.log(config);
var hsqldb=new jdbc(config);
hsqldb.status();
hsqldb.initialize(config, function(err, res) {
console.log("Connection opened successfully!");
if (err) {
console.log("Error during Initializing");
console.log(err);
}
});
exports.list = function(req, res){
hsqldb.open(function(err, conn) {
if (conn) {
// SELECT statements are called with executeQuery
hsqldb.executeQuery("select * from schema.table", function(err, results) {
if (err) {
console.log(err);
} else if (results) {
console.log(results);
res.type('text/plain'); // set content-type
res.send(results);
}
hsqldb.close(function(err) {
if(err) {
console.log(err);
} else {
console.log("Connection closed successfully!");
}
});
}
);
}
});
// res.send("respond with a resource");
};
Error:
C:\Users\ABCD\node_modules\jdbc\lib\pool.js:97
return callback(null);
^
TypeError: object is not a function
at C:\Users\ABCD\node_modules\jdbc\lib\pool.js:97:12
at C:\Users\ABCD\node_modules\jdbc\node_modules\async\lib\async.js:52:16
at C:\Users\ABCD\node_modules\jdbc\node_modules\async\lib\async.js:363:13
at C:\Users\ABCD\node_modules\jdbc\node_modules\async\lib\async.js:52:16
at done (C:\Users\ABCD\node_modules\jdbc\node_modules\async\lib\async.js:243:17)
at C:\Users\ABCD\node_modules\jdbc\node_modules\async\lib\async.js:44:16
at C:\Users\ABCD\node_modules\jdbc\node_modules\async\lib\async.js:360:17
at C:\Users\ABCD\node_modules\jdbc\lib\pool.js:90:7
at C:\Users\ABCD\node_modules\jdbc\lib\pool.js:28:14
at C:\Users\ABCD\node_modules\jdbc\lib\drivermanager.js:26:18
Similar Post: Redshift data access from node jdbc
Could Some one please let me know what went wrong here.Is it some thing wrong with the code? or is it with the modules not installed correctly. I feel it is due to modules not installed correctly since I saw few errors while installing the jdbc module.
Please let me know the steps to be taken to install jdbc module in node js.
The node-jdbc API has been completely reworked since version 0.1.1 was released, so any old documentation relating to older versions of node-jdbc, including my answer to the question linked above, is no longer helpful.
Using node-jdbc 0.1.1, the code to connect to your database should look something like the following:
var jdbc = require('jdbc');
var jinst = require('jdbc/lib/jinst');
if (!jinst.isJvmCreated()) {
jinst.addOption("-Xrs");
jinst.setupClasspath(['../RedshiftJDBC41-1.1.6.1006.jar']);
}
var config = {
drivername: 'com.amazon.redshift.jdbc41.Driver',
url: 'jdbc:redshift://exampleCluster.abcdkcmf0ug1.us-west-2.redshift.amazonaws.com:5439/xyz',
properties: {
"user": "XXXXX",
"password": "XXXXX"
}
};
var hsqldb=new JDBC(config);
hsqldb.status();
hsqldb.initialize(function(err, res) {
/* ... */
});
In particular:
The classpath is now configured in a jinst object and has moved out of the configuration.
properties is now an object, not an array of 2-element arrays as it was in version 0.0.15.
The initialize method now takes only one parameter, the callback. It no longer takes the configuration as a parameter as that was provided in the constructor.
node-jdbc supposedly supports specifying the user and password properties in the config object, which avoids using properties for this. However, I would recommend avoiding this as it does not work. If you try it, you may well get errors about the username and password not being valid, even if you've typed them in correctly.
The reason you are getting the somewhat unintelligible error is because you were passing the configuration object to the initialize method, which was expecting only a callback. The error doesn't get noticed until Node tries to 'call' the configuration object as if it were a function, which of course fails.

node module.export and recursion

I am working on a node app that essentially is a simple AWS SQS poller that should sit and listen to new items in different queues.
Here is my module.export:
module.exports = {
readMessage: function(qParams, qType, tableName) {
logger.debug(qType);
SQS.receiveMessage(qParams, handleSqsResponse);
function handleSqsResponse (err, data) {
if(err) logger.error("handleSqsResponse error:" + err);
if (data && data.Messages) {
data.Messages.forEach(processMessage)
readMessage(); // continue reading until draining the queue (or UPTIME reached)
}
else{
logger.debug("no data in sqs.");
// process.exit();
}
}
// 'processing' is mainly writing to logs using winston. Could add here any transformations and transmission to remote systems
function processMessage(sqsMessage){
// Parse sqs messag
var msgObj = JSON.parse(sqsMessage.Body);
// Process
logger.info(msgObj.Message);
_.extend(qParams, { "ReceiptHandle": sqsMessage.ReceiptHandle });
dbMap[qType](msgObj, qParams, tableName);
}
}
}
The issue I am running into is when I attempt to call readMessage(); again. I get the error of ReferenceError: readMessage is not defined
module.exports is a plain object that is exposed to outer modules that has a method readMessage. readMessage() should be module.exports.readMessage().
Also i would suggest creating a variable and then exporting that:
var obj = {
readMessage: function(qParams, qType, tableName) {
logger.debug(qType);
SQS.receiveMessage(qParams, handleSqsResponse);
function handleSqsResponse (err, data) {
if(err) logger.error("handleSqsResponse error:" + err);
if (data && data.Messages) {
data.Messages.forEach(processMessage)
obj.readMessage(); // continue reading until draining the queue (or UPTIME reached)
}
else{
logger.debug("no data in sqs.");
// process.exit();
}
}
// 'processing' is mainly writing to logs using winston. Could add here any transformations and transmission to remote systems
function processMessage(sqsMessage){
// Parse sqs messag
var msgObj = JSON.parse(sqsMessage.Body);
// Process
logger.info(msgObj.Message);
_.extend(qParams, { "ReceiptHandle": sqsMessage.ReceiptHandle });
dbMap[qType](msgObj, qParams, tableName);
}
}
}
module.exports = obj;
Please note that I only responded to the question you specifically asked. I didn't take into account any architectural issue associate with the code.
function functionName(has = false){
var total = 0;
if(has){
functionName(true)
} else {
// Todo
}
}
module.exports.functionName = functionName;

How should I create custom modules that require a mongodb connection?

I'm working on a web app with nodejs, express, and mongodb.
In my 'main' file where I listed for API calls, I include a Users class that has methods like Users.authenticate(userObject, callback), and Users.getById(userId, callback).
Sorry for this long code snippet. It's just a snippet of my users class.
function Users (db) {
if (!db) {
return {'message': 'creating an instance of Users requires a database'}
} else {
this.db = db;
return this;
}
}
Users.prototype.authenticate = function (user, callback) {
if (!user.username) {
return {'message': 'Users.authenticate(user, callback) requires user.username'};
} else if (!user.password) {
return {'message': 'Users.authenticate(user, callback) requires user.password'};
} else if (!callback) {
return {'message': 'Users.authenticate(user, callback) requires callback(err, user)'};
}
this.db.collection('users', function (err, collection) {
if (err) {return {'message': 'could not open users collection'}};
/* query for the user argument */
collection.findOne(user, function (err, doc) {
if (!err) {
if (!doc) {
callback({'message': 'user does not exist'}, null);
} else {
callback(null, doc);
}
} else {
callback({'message': 'error finding user'}, null);
}
});
});
};
exports.Users = Users;
That's it
I pass an open DB connection to my Users class, and make a call like the following:
var server = new mongo.Server('localhost', '27017', {auto_reconnect: true});
var db = new mongo.Db('supportdash', server, {"fsync": true});
// open connection to be, init users
db.open(function (err, db) {
var users = new Users(db);
users.authenticate({"username": "admin", "password": "password"}, function (err, user) {
// do something with the error, or user object
});
});
Now for my questions
Should I be passing an open db connection, or should I be passing the info needed (localhost, port, database name) for the Users class to manage its own connection?
I tried to set up testing with jasmine-node, but I ended up with a lot of problems with async database calls. I wanted to add a user, then test that Users.authenticate was working. I used Jasmines runs() and waitsfor() async helpers, but I could not get it to work. I then ran into an issue that took me a while to debug (with a different class), and testing would have saved me a lot of time. Any advice on how I would test my classes that interact with a mongodb database?

Cannot enqueue Handshake after invoking quit

I have implemented the following code:
module.exports = {
getDataFromUserGps: function(callback)
{
connection.connect();
connection.query("SELECT * FROM usergps",
function(err, results, fields) {
if (err) return callback(err, null);
return callback(null, results);
}
);
connection.end();
},
loginUser: function(login, pass, callback)
{
connection.connect();
connection.query(
"SELECT id FROM users WHERE login = ? AND pass = ?",
[login, pass],
function(err, results, fields)
{
if (err) return callback(err, null);
return callback(null, results);
}
);
connection.end();
},
getUserDetails: function(userid, callback)
{
connection.connect();
connection.query(
"SELECT * FROM userProfilDetails LEFT JOIN tags ON userProfilDetails.userId = tags.userId WHERE userProfilDetails.userid = ?",
[userid],
function(err, results, fields)
{
if (err) return callback(err, null);
return callback(null, results);
}
);
connection.end();
},
addTags: function(userId, tags)
{
connection.connect();
connection.query(
"INSERT INTO tag (userId, tag) VALUES (?, ?)",
[userId, tags],
function(err, results, fields)
{
if (err) throw err;
}
)
connection.end();
}
}
Everything works great only for the first time. If I want to "use" the query for the second time I get the following error:
Cannot enqueue Handshake after invoking quit
I have tried not to .end() connections but it didn't help.
How can I fix this issue?
If you using the node-mysql module, just remove the .connect and .end. Just solved the problem myself. Apparently they pushed in unnecessary code in their last iteration that is also bugged. You don't need to connect if you have already ran the createConnection call
According to:
Fixing Node Mysql "Error: Cannot enqueue Handshake after invoking quit.":
http://codetheory.in/fixing-node-mysql-error-cannot-enqueue-handshake-after-invoking-quit/
TL;DR You need to establish a new connection by calling the createConnection method after every disconnection.
and
Note: If you're serving web requests, then you shouldn't be ending connections on every request. Just create a connection on server
startup and use the connection/client object to query all the time.
You can listen on the error event to handle server disconnection and
for reconnecting purposes. Full code
here.
From:
Readme.md - Server disconnects:
https://github.com/felixge/node-mysql#server-disconnects
It says:
Server disconnects
You may lose the connection to a MySQL server due to network problems,
the server timing you out, or the server crashing. All of these events
are considered fatal errors, and will have the err.code =
'PROTOCOL_CONNECTION_LOST'. See the Error
Handling section for more information.
The best way to handle such unexpected disconnects is shown below:
function handleDisconnect(connection) {
connection.on('error', function(err) {
if (!err.fatal) {
return;
}
if (err.code !== 'PROTOCOL_CONNECTION_LOST') {
throw err;
}
console.log('Re-connecting lost connection: ' + err.stack);
connection = mysql.createConnection(connection.config);
handleDisconnect(connection);
connection.connect();
});
}
handleDisconnect(connection);
As you can see in the example above, re-connecting a connection is
done by establishing a new connection. Once terminated, an existing
connection object cannot be re-connected by design.
With Pool, disconnected connections will be removed from the pool
freeing up space for a new connection to be created on the next
getConnection call.
I have tweaked the function such that every time a connection is needed, an initializer function adds the handlers automatically:
function initializeConnection(config) {
function addDisconnectHandler(connection) {
connection.on("error", function (error) {
if (error instanceof Error) {
if (error.code === "PROTOCOL_CONNECTION_LOST") {
console.error(error.stack);
console.log("Lost connection. Reconnecting...");
initializeConnection(connection.config);
} else if (error.fatal) {
throw error;
}
}
});
}
var connection = mysql.createConnection(config);
// Add handlers.
addDisconnectHandler(connection);
connection.connect();
return connection;
}
Initializing a connection:
var connection = initializeConnection({
host: "localhost",
user: "user",
password: "password"
});
Minor suggestion: This may not apply to everyone but I did run into a minor issue relating to scope. If the OP feels this edit was unnecessary then he/she can choose to remove it. For me, I had to change a line in initializeConnection, which was var connection = mysql.createConnection(config); to simply just
connection = mysql.createConnection(config);
The reason being that if connection is a global variable in your program, then the issue before was that you were making a new connection variable when handling an error signal. But in my nodejs code, I kept using the same global connection variable to run queries on, so the new connection would be lost in the local scope of the initalizeConnection method. But in the modification, it ensures that the global connection variable is reset This may be relevant if you're experiencing an issue known as
Cannot enqueue Query after fatal error
after trying to perform a query after losing connection and then successfully reconnecting. This may have been a typo by the OP, but I just wanted to clarify.
I had the same problem and Google led me here. I agree with #Ata that it's not right to just remove end(). After further Googling, I think using pooling is a better way.
node-mysql doc about pooling
It's like this:
var mysql = require('mysql');
var pool = mysql.createPool(...);
pool.getConnection(function(err, connection) {
connection.query( 'bla bla', function(err, rows) {
connection.release();
});
});
Do not connect() and end() inside the function. This will cause problems on repeated calls to the function. Make the connection only
var connection = mysql.createConnection({
host: 'localhost',
user: 'node',
password: 'node',
database: 'node_project'
})
connection.connect(function(err) {
if (err) throw err
});
once and reuse that connection.
Inside the function
function insertData(name,id) {
connection.query('INSERT INTO members (name, id) VALUES (?, ?)', [name,id], function(err,result) {
if(err) throw err
});
}
AWS Lambda functions
Use mysql.createPool() with connection.destroy()
This way, new invocations use the established pool, but don't keep the function running. Even though you don't get the full benefit of pooling (each new connection uses a new connection instead of an existing one), it makes it so that a second invocation can establish a new connection without the previous one having to be closed first.
Regarding connection.end()
This can cause a subsequent invocation to throw an error. The invocation will still retry later and work, but with a delay.
Regarding mysql.createPool() with connection.release()
The Lambda function will keep running until the scheduled timeout, as there is still an open connection.
Code example
const mysql = require('mysql');
const pool = mysql.createPool({
connectionLimit: 100,
host: process.env.DATABASE_HOST,
user: process.env.DATABASE_USER,
password: process.env.DATABASE_PASSWORD,
});
exports.handler = (event) => {
pool.getConnection((error, connection) => {
if (error) throw error;
connection.query(`
INSERT INTO table_name (event) VALUES ('${event}')
`, function(error, results, fields) {
if (error) throw error;
connection.destroy();
});
});
};
I think this issue is similar to mine:
Connect to MySQL
End MySQL service (should not quit node script)
Start MySQL service, Node reconnects to MySQL
Query the DB -> FAIL (Cannot enqueue Query after fatal error.)
I solved this issue by recreating a new connection with the use of promises (q).
mysql-con.js
'use strict';
var config = require('./../config.js');
var colors = require('colors');
var mysql = require('mysql');
var q = require('q');
var MySQLConnection = {};
MySQLConnection.connect = function(){
var d = q.defer();
MySQLConnection.connection = mysql.createConnection({
host : 'localhost',
user : 'root',
password : 'password',
database : 'database'
});
MySQLConnection.connection.connect(function (err) {
if(err) {
console.log('Not connected '.red, err.toString().red, ' RETRYING...'.blue);
d.reject();
} else {
console.log('Connected to Mysql. Exporting..'.blue);
d.resolve(MySQLConnection.connection);
}
});
return d.promise;
};
module.exports = MySQLConnection;
mysqlAPI.js
var colors = require('colors');
var mysqlCon = require('./mysql-con.js');
mysqlCon.connect().then(function(con){
console.log('connected!');
mysql = con;
mysql.on('error', function (err, result) {
console.log('error occurred. Reconneting...'.purple);
mysqlAPI.reconnect();
});
mysql.query('SELECT 1 + 1 AS solution', function (err, results) {
if(err) console.log('err',err);
console.log('Works bro ',results);
});
});
mysqlAPI.reconnect = function(){
mysqlCon.connect().then(function(con){
console.log("connected. getting new reference");
mysql = con;
mysql.on('error', function (err, result) {
mysqlAPI.reconnect();
});
}, function (error) {
console.log("try again");
setTimeout(mysqlAPI.reconnect, 2000);
});
};
I hope this helps.
inplace of connection.connect(); use -
if(!connection._connectCalled )
{
connection.connect();
}
if it is already called then connection._connectCalled =true,
& it will not execute connection.connect();
note - don't use connection.end();
SOLUTION: to prevent this error(for AWS LAMBDA):
In order to exit of "Nodejs event Loop" you must end the connection, and then reconnect. Add the next code to invoke the callback:
connection.end( function(err) {
if (err) {console.log("Error ending the connection:",err);}
// reconnect in order to prevent the"Cannot enqueue Handshake after invoking quit"
connection = mysql.createConnection({
host : 'rds.host',
port : 3306,
user : 'user',
password : 'password',
database : 'target database'
});
callback(null, {
statusCode: 200,
body: response,
});
});
If you're trying to get a lambda, I found that ending the handler with context.done() got the lambda to finish. Before adding that 1 line, It would just run and run until it timed out.
You can use
debug: false,
Example:
//mysql connection
var dbcon1 = mysql.createConnection({
host: "localhost",
user: "root",
password: "",
database: "node5",
debug: false,
});
A little digging showed that I wasn't closing the connection at all.
So added this code before I opened up connection and when I was done with database manipulation
connection.end()
connection = mysql.createConnection(
// database connection details
)
connection.connect(function (err) {
if (!err) {
console.log("Connected!");
var sql = `Select something from my_heart;`
connection.query(sql, function (err, result) {
if (!err) {
console.log("1 record inserted");
res.send("Recieved")
} else {
console.log(err.sqlMessage)
res.send("error")
}
});
}
})
Just use connection.connect() once outside of module.exports. It should be connect() once when node server is initialised, not in every request.
You can do this in this way :--
const connection = sql.createConnection({
host: "****",
user: "****",
password: "*****",
database: "****"
})
connection.connect((error) => {
if( error ) throw new Error(error)
})
module.exports = {
getDataFromUserGps: function(callback)
{
connection.query("SELECT * FROM usergps",
function(err, results, fields) {
if (err) return callback(err, null);
return callback(null, results);
}
);
},
****
****
****
}

Resources