I try to move datastorage in my node app to mongo db. But I have a simple problem with mongo db.
I have a button, clicked in website, will call /datastore
app.post('/datastore', (req, res) => {
client.connect(err => {
var dbo = client.db("test");
dbo.listCollections().toArray(function(err, items){
if (err) throw err;
console.log(items);
if (items.length == 0)
console.log("No collections in database")
});
client.close();
});
});
It works fine for the first time I click the button. But if I click the second time the button I get errors messages:
the options [servers] is not supported the options [caseTranslate] is
not supported the options [dbName] is not supported the options
[credentials] is not supported
/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb/lib/utils.js:132
throw err;
^
MongoError: Topology was destroyed
at initializeCursor (/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb-core/lib/cursor.js:596:25)
at nextFunction (/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb-core/lib/cursor.js:456:12)
at CommandCursor.Cursor.next (/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb-core/lib/cursor.js:766:3)
at CommandCursor.Cursor._next (/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb/lib/cursor.js:216:36)
at fetchDocs (/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb/lib/operations/cursor_ops.js:217:12)
at toArray (/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb/lib/operations/cursor_ops.js:247:3)
at executeOperation (/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb/lib/utils.js:416:24)
at CommandCursor.Cursor.toArray (/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb/lib/cursor.js:829:10)
at client.connect.err (/Users/ingofoerster/Downloads/development/testrunner/start.js:256:35)
at result (/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb/lib/utils.js:410:17)
at executeCallback (/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb/lib/utils.js:402:9)
at err (/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb/lib/operations/mongo_client_ops.js:286:5)
at connectCallback (/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb/lib/operations/mongo_client_ops.js:265:5)
at topology.connect (/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb/lib/operations/mongo_client_ops.js:417:5)
at ReplSet.connectHandler (/Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb/lib/topologies/replset.js:343:9)
at Object.onceWrapper (events.js:281:20)
at ReplSet.emit (events.js:193:13)
at /Users/ingofoerster/Downloads/development/testrunner/node_modules/mongodb-core/lib/topologies/replset.js:786:18
at processTicksAndRejections (internal/process/task_queues.js:79:9)
I cannot explain why this happens, because I have the client.close() in my code. Any idea why I can not call the function more than one time?
When you clicked the button for the first time, you are able to get the result as expected, but after getting the result you are also closing the connection by calling
client.close(); , which is not letting MongoDB reconnect for the second time.
Ideally, you should reuse the existing connection instead of calling connect method for each API call.
Connection pooling example is taken from here
var express = require('express');
var mongodb = require('mongodb');
var app = express();
var MongoClient = require('mongodb').MongoClient;
var db;
// Initialize connection once
MongoClient.connect("mongodb://localhost:27017/integration_test", function(err, database) {
if(err) throw err;
db = database;
// Start the application after the database connection is ready
app.listen(3000);
console.log("Listening on port 3000");
});
// Reuse database object in request handlers
app.get("/", function(req, res) {
db.collection("replicaset_mongo_client_collection").find({}, function(err, docs) {
docs.each(function(err, doc) {
if(doc) {
console.log(doc);
}
else {
res.end();
}
});
});
});
Executing a find query to my MongoDB DB seems to reset the connection and make the node server crash.
My server handles socket events like this:
io.sockets.on('connection', function(socket) {
MongoClient.connect(url, function(err, db) {
if (err)
console.log('Error');
console.log('Connection established to', url);
var collection = db.collection('spedizioni');
socket.on('adminReq', function() {
handlers.handleAdmin(collection, socket);
});
});
});
the handleAdmin function is:
function handleAdmin(collection, socket) {
console.log('Admin event');
collection.find(null, function(err, raw) {
console.log('Find function');
console.log(raw);
if (err){
socket.emit('err');
console.log('Error function');
}
if (raw) {
socket.emit('adminRes', raw);
console.log('Response function');
} else {
socket.emit('adminNull');
console.log('Null function');
}
});
}
I want the query to return all items on the database; as per the MongoDB manual, I do that by executing a find query without a parameter.
I tried omitting null or using {} as first parameter but nothing changes.
Upon pressing the button to generate the adminReq event, the 'Connection to DB' string is printed on console and the firefox console signals a NEW connection to socket was estabilished; my client script connects at document.load once.
Below is the node console output after that; as you can see the query is executed; looking at the 'raw' output it seems failed attempts were made.
err is null and there is nothing else to output.
Looking at other answers about the 'maximum call stack' exceeded it seems it is caused by a recursive function usually, but it's not the case here.
http://pastebin.com/0xv1qcHn
Why is this the output and not the query result? Why is the connection reset?
A very similar function is working fine and the syntax for returning the whole DB seems correct, feels I am missing something very obvious...
not sure if you should can use null, but i think an empty object should work
you need to convert your result into an array
collection.find({}).toArray(function(err, raw) {
console.log('Find function');
console.log(raw);
if (err){
socket.emit('err');
console.log('Error function');
}
if (raw) {
socket.emit('adminRes', raw);
console.log('Response function');
} else {
socket.emit('adminNull');
console.log('Null function');
}
});
Trying to drop all docs in all collections before unit tests run...
var collections = mongoose.connection.collections;
async.eachSeries(_.keys(collections), function(key, cb){
collections[key].remove(function(){
//Never gets here, doesn't drop the collection and doesn't error.
cb();
});
}
But the callback in remove() never gets fired.
I've logged out collections[key] and it does resolve to a collection.
No errors, but it times out as it never runs the callback.
Ive tried looping the Models as well and calling that remove but its the same issue.
What am I doing wrong here?? Any logs I could look at?
You could try using the drop method:
var async = require("async"),
_ = require("underscore"),
collections = _.keys(mongoose.connection.collections);
async.forEach(collections, function(key, done) {
var collection = mongoose.connection.collections[key]
collection.drop(function(err) {
if (err && err.message != "ns not found") {
done(err);
} else {
done(null);
}
})
}, callback);
EDIT: Try the following:
var collections = mongoose.connection.collections;
async.eachSeries(_.keys(collections), function(key, cb){
mongoose.connection.db.collection(key, function(err, col) {
col.remove({}, function(){
cb();
});
})
}
Unrelated issue, it wasn't connecting to the DB in the test environment. But no errors were reported, and it had a valid list of collections due to mongoose models.
I added the following to the test set up to log out the errors and other info to help find these issues in the future...
mongoose.connection.on('connected', function () {
console.log('Mongoose default connection open to ' + config.db.url);
});
mongoose.connection.on('error',function (err) {
console.log('Mongoose default connection error: ' + err);
});
mongoose.connection.on('disconnected', function () {
console.log('Mongoose default connection disconnected');
});
I'm a newbie as far as node is concerned, so what i am asking here might be plain out stupid, so bear with me.
Below is the code for a node module that connects to a given mongodb database, my problem is on line19 where i am trying to throw the error in case the connection to the db server cannot be made or the db server is down, but node complains , please advise.
Code:-
var dbinit_func = function(db_name){
try{
// require mongoose , if it's not there
// throw an exception and but out
var mongoose = require('mongoose');
}
catch(err){
throw "Error Mongoose Not Found"
}
try{
// connect to the db
mongoose.connect("mongodb://localhost/" + db_name);
// get a reference to the connection object
var db_connection = mongoose.connection;
// subscribe to events on the connection object
db_connection.on('error', function(err){
// holy cow "A Connection Error", shout it out loud
// and but out
throw new Error(err.message);--> This is where the problem Occurs
});
// bind to the connection open event , we just need to
// do it once , so we use the once method on the
// connection object
db_connection.once('open', function(){})
}
catch(err){
// we got an error most probably a connection error
// so we but out from here
throw "Connection Error";
}
}
module.exports = dbinit_func;
Message spitted by Node:-
/Users/tristan625/projects/node_projs/school_web/models/db.js:19
throw new Error(err.message);
^
Your code is written in a synchronous style, as seen by the try/catch blocks. What you must understand, is that the mongoose.connect function is asynchronous. When you call throw new Error(), that call will occur inside of a callback, which is outside the scope of your try/catch blocks, hence you can't handle the error using them.
Your dbinit_func must be written to allow for a callback
Here's a pretty standard way of doing what you're doing
function dbinit_func(db_name, callback) {
var mongoose = require('mongoose');
mongoose.connect('mongodb://localhost/ + db_name', { auto_reconnect: true });
var db = mongoose.connection;
db.once('error', function(err){
//trigger callback with an error
callback(err);
});
db.once('open', function() {
//trigger callback with success
callback(null, 'DB Connection is open');
});
}
module.exports = dbinit_func;
// to call this function, you'd want to do something like this
var dbinit = require('/path/to/your/module');
dbinit('mydatabasename', function(err, res){
// at this point, there will either be an error
if (err) console.log(err.message);
// else it will be success
console.log(res);
});
I have implemented the following code:
module.exports = {
getDataFromUserGps: function(callback)
{
connection.connect();
connection.query("SELECT * FROM usergps",
function(err, results, fields) {
if (err) return callback(err, null);
return callback(null, results);
}
);
connection.end();
},
loginUser: function(login, pass, callback)
{
connection.connect();
connection.query(
"SELECT id FROM users WHERE login = ? AND pass = ?",
[login, pass],
function(err, results, fields)
{
if (err) return callback(err, null);
return callback(null, results);
}
);
connection.end();
},
getUserDetails: function(userid, callback)
{
connection.connect();
connection.query(
"SELECT * FROM userProfilDetails LEFT JOIN tags ON userProfilDetails.userId = tags.userId WHERE userProfilDetails.userid = ?",
[userid],
function(err, results, fields)
{
if (err) return callback(err, null);
return callback(null, results);
}
);
connection.end();
},
addTags: function(userId, tags)
{
connection.connect();
connection.query(
"INSERT INTO tag (userId, tag) VALUES (?, ?)",
[userId, tags],
function(err, results, fields)
{
if (err) throw err;
}
)
connection.end();
}
}
Everything works great only for the first time. If I want to "use" the query for the second time I get the following error:
Cannot enqueue Handshake after invoking quit
I have tried not to .end() connections but it didn't help.
How can I fix this issue?
If you using the node-mysql module, just remove the .connect and .end. Just solved the problem myself. Apparently they pushed in unnecessary code in their last iteration that is also bugged. You don't need to connect if you have already ran the createConnection call
According to:
Fixing Node Mysql "Error: Cannot enqueue Handshake after invoking quit.":
http://codetheory.in/fixing-node-mysql-error-cannot-enqueue-handshake-after-invoking-quit/
TL;DR You need to establish a new connection by calling the createConnection method after every disconnection.
and
Note: If you're serving web requests, then you shouldn't be ending connections on every request. Just create a connection on server
startup and use the connection/client object to query all the time.
You can listen on the error event to handle server disconnection and
for reconnecting purposes. Full code
here.
From:
Readme.md - Server disconnects:
https://github.com/felixge/node-mysql#server-disconnects
It says:
Server disconnects
You may lose the connection to a MySQL server due to network problems,
the server timing you out, or the server crashing. All of these events
are considered fatal errors, and will have the err.code =
'PROTOCOL_CONNECTION_LOST'. See the Error
Handling section for more information.
The best way to handle such unexpected disconnects is shown below:
function handleDisconnect(connection) {
connection.on('error', function(err) {
if (!err.fatal) {
return;
}
if (err.code !== 'PROTOCOL_CONNECTION_LOST') {
throw err;
}
console.log('Re-connecting lost connection: ' + err.stack);
connection = mysql.createConnection(connection.config);
handleDisconnect(connection);
connection.connect();
});
}
handleDisconnect(connection);
As you can see in the example above, re-connecting a connection is
done by establishing a new connection. Once terminated, an existing
connection object cannot be re-connected by design.
With Pool, disconnected connections will be removed from the pool
freeing up space for a new connection to be created on the next
getConnection call.
I have tweaked the function such that every time a connection is needed, an initializer function adds the handlers automatically:
function initializeConnection(config) {
function addDisconnectHandler(connection) {
connection.on("error", function (error) {
if (error instanceof Error) {
if (error.code === "PROTOCOL_CONNECTION_LOST") {
console.error(error.stack);
console.log("Lost connection. Reconnecting...");
initializeConnection(connection.config);
} else if (error.fatal) {
throw error;
}
}
});
}
var connection = mysql.createConnection(config);
// Add handlers.
addDisconnectHandler(connection);
connection.connect();
return connection;
}
Initializing a connection:
var connection = initializeConnection({
host: "localhost",
user: "user",
password: "password"
});
Minor suggestion: This may not apply to everyone but I did run into a minor issue relating to scope. If the OP feels this edit was unnecessary then he/she can choose to remove it. For me, I had to change a line in initializeConnection, which was var connection = mysql.createConnection(config); to simply just
connection = mysql.createConnection(config);
The reason being that if connection is a global variable in your program, then the issue before was that you were making a new connection variable when handling an error signal. But in my nodejs code, I kept using the same global connection variable to run queries on, so the new connection would be lost in the local scope of the initalizeConnection method. But in the modification, it ensures that the global connection variable is reset This may be relevant if you're experiencing an issue known as
Cannot enqueue Query after fatal error
after trying to perform a query after losing connection and then successfully reconnecting. This may have been a typo by the OP, but I just wanted to clarify.
I had the same problem and Google led me here. I agree with #Ata that it's not right to just remove end(). After further Googling, I think using pooling is a better way.
node-mysql doc about pooling
It's like this:
var mysql = require('mysql');
var pool = mysql.createPool(...);
pool.getConnection(function(err, connection) {
connection.query( 'bla bla', function(err, rows) {
connection.release();
});
});
Do not connect() and end() inside the function. This will cause problems on repeated calls to the function. Make the connection only
var connection = mysql.createConnection({
host: 'localhost',
user: 'node',
password: 'node',
database: 'node_project'
})
connection.connect(function(err) {
if (err) throw err
});
once and reuse that connection.
Inside the function
function insertData(name,id) {
connection.query('INSERT INTO members (name, id) VALUES (?, ?)', [name,id], function(err,result) {
if(err) throw err
});
}
AWS Lambda functions
Use mysql.createPool() with connection.destroy()
This way, new invocations use the established pool, but don't keep the function running. Even though you don't get the full benefit of pooling (each new connection uses a new connection instead of an existing one), it makes it so that a second invocation can establish a new connection without the previous one having to be closed first.
Regarding connection.end()
This can cause a subsequent invocation to throw an error. The invocation will still retry later and work, but with a delay.
Regarding mysql.createPool() with connection.release()
The Lambda function will keep running until the scheduled timeout, as there is still an open connection.
Code example
const mysql = require('mysql');
const pool = mysql.createPool({
connectionLimit: 100,
host: process.env.DATABASE_HOST,
user: process.env.DATABASE_USER,
password: process.env.DATABASE_PASSWORD,
});
exports.handler = (event) => {
pool.getConnection((error, connection) => {
if (error) throw error;
connection.query(`
INSERT INTO table_name (event) VALUES ('${event}')
`, function(error, results, fields) {
if (error) throw error;
connection.destroy();
});
});
};
I think this issue is similar to mine:
Connect to MySQL
End MySQL service (should not quit node script)
Start MySQL service, Node reconnects to MySQL
Query the DB -> FAIL (Cannot enqueue Query after fatal error.)
I solved this issue by recreating a new connection with the use of promises (q).
mysql-con.js
'use strict';
var config = require('./../config.js');
var colors = require('colors');
var mysql = require('mysql');
var q = require('q');
var MySQLConnection = {};
MySQLConnection.connect = function(){
var d = q.defer();
MySQLConnection.connection = mysql.createConnection({
host : 'localhost',
user : 'root',
password : 'password',
database : 'database'
});
MySQLConnection.connection.connect(function (err) {
if(err) {
console.log('Not connected '.red, err.toString().red, ' RETRYING...'.blue);
d.reject();
} else {
console.log('Connected to Mysql. Exporting..'.blue);
d.resolve(MySQLConnection.connection);
}
});
return d.promise;
};
module.exports = MySQLConnection;
mysqlAPI.js
var colors = require('colors');
var mysqlCon = require('./mysql-con.js');
mysqlCon.connect().then(function(con){
console.log('connected!');
mysql = con;
mysql.on('error', function (err, result) {
console.log('error occurred. Reconneting...'.purple);
mysqlAPI.reconnect();
});
mysql.query('SELECT 1 + 1 AS solution', function (err, results) {
if(err) console.log('err',err);
console.log('Works bro ',results);
});
});
mysqlAPI.reconnect = function(){
mysqlCon.connect().then(function(con){
console.log("connected. getting new reference");
mysql = con;
mysql.on('error', function (err, result) {
mysqlAPI.reconnect();
});
}, function (error) {
console.log("try again");
setTimeout(mysqlAPI.reconnect, 2000);
});
};
I hope this helps.
inplace of connection.connect(); use -
if(!connection._connectCalled )
{
connection.connect();
}
if it is already called then connection._connectCalled =true,
& it will not execute connection.connect();
note - don't use connection.end();
SOLUTION: to prevent this error(for AWS LAMBDA):
In order to exit of "Nodejs event Loop" you must end the connection, and then reconnect. Add the next code to invoke the callback:
connection.end( function(err) {
if (err) {console.log("Error ending the connection:",err);}
// reconnect in order to prevent the"Cannot enqueue Handshake after invoking quit"
connection = mysql.createConnection({
host : 'rds.host',
port : 3306,
user : 'user',
password : 'password',
database : 'target database'
});
callback(null, {
statusCode: 200,
body: response,
});
});
If you're trying to get a lambda, I found that ending the handler with context.done() got the lambda to finish. Before adding that 1 line, It would just run and run until it timed out.
You can use
debug: false,
Example:
//mysql connection
var dbcon1 = mysql.createConnection({
host: "localhost",
user: "root",
password: "",
database: "node5",
debug: false,
});
A little digging showed that I wasn't closing the connection at all.
So added this code before I opened up connection and when I was done with database manipulation
connection.end()
connection = mysql.createConnection(
// database connection details
)
connection.connect(function (err) {
if (!err) {
console.log("Connected!");
var sql = `Select something from my_heart;`
connection.query(sql, function (err, result) {
if (!err) {
console.log("1 record inserted");
res.send("Recieved")
} else {
console.log(err.sqlMessage)
res.send("error")
}
});
}
})
Just use connection.connect() once outside of module.exports. It should be connect() once when node server is initialised, not in every request.
You can do this in this way :--
const connection = sql.createConnection({
host: "****",
user: "****",
password: "*****",
database: "****"
})
connection.connect((error) => {
if( error ) throw new Error(error)
})
module.exports = {
getDataFromUserGps: function(callback)
{
connection.query("SELECT * FROM usergps",
function(err, results, fields) {
if (err) return callback(err, null);
return callback(null, results);
}
);
},
****
****
****
}