I am writing a web application that uses asynchronous database requests as a part of the api. Currently, I have an async express route that awaits function returns from async functions. Both of these functions return booleans and both query the database. One works correctly, however the second one does not.
Here is the MongoClient setup:
const MongoClient = require('mongodb').MongoClient;
const uri = config.uri; // Contains custom url for accessing database
const client = new MongoClient(uri, { useUnifiedTopology: true}, { useNewUrlParser: true }, { connectTimeoutMS: 30000 }, { keepAlive: 1});
where config is from a file imported as.
const config = require("./config.js");
and functions properly.
Here is the express setup:
app.post("/signup", async function(request, response) {
log("POST request at /signup");
log("BEFORE UNIQUE USER");
const isUniqueUser = await validateUniqueUser(request.body.email, request.body.password);
log(isUniqueUser);
const status = {
status: null
};
if (isUniqueUser) {
log("AFTER UNIQUE USER");
let userCreated = await createPracticeProfile(request.body.email, request.body.password);
log("user created: " + userCreated);
if (userCreated) {
status.status = "user_created";
}
response.json(status);
} else {
response.json(status);
}
console.log("********************************end");
});
The console outputs:
BEFORE UNIQUE USER
true (which it should be)
AFTER UNIQUE USER
MongoError: Topology is closed.
user created: undefined
***...***end
Here is the function for validating that a user is unique:
/* VALIDATE_UNIQUE_USER
USE: ensure user does not have existing profile
PARAMS: email (string), password (string)
RETURN: isUniqueUser (bool)
*/
async function validateUniqueUser(email, password) {
// connect to database
const database = await client.connect().catch(err => {
log("ERROR while connecting to database at: validateUniqueUser");
console.log(err);
client.close();
});
// database connection failed
if (!database) {
return false;
}
// connection successful => find user
let user;
try {
user = await database.db("guitar-practice-suite").collection("users").findOne({email: email});
} catch(err) {
log("ERROR while finding user in database at: validateUniqueUser");
console.log(err);
client.close();
return false;
} finally {
client.close();
// user not found (unique)
if (user === null || user === undefined) {
return true;
}
return false;
}
}
Here is the function for inserting the user into the collections:
/* CREATE_PRACTICE_PROFILE
USE: insert a practice profile into the database
PARAMS: email (string), password (string)
RETURN: userCreated (bool)
*/
async function createPracticeProfile(email, password) {
// hash password
let hashedPassword;
try {
hashedPassword = await new Promise((resolve, reject) => {
bcrypt.hash(password, null, null, function(err, hash) {
if (err) {
reject(err);
}
resolve(hash)
});
});
} catch(err) {
log("ERROR while hashing password at: createPracticeProfile");
console.log(err);
return false;
}
// connect to database
const database = await client.connect().catch(err => {
log("ERROR while connecting to database at: validateUniqueUser");
console.log(err);
client.close();
});
// database connection failed
if (!database) {
return false;
}
// database connection successful => insert user into database
let insertUserToUsers;
let insertUserToExercises;
let insertUserToCustomExercises;
try {
insertUserToUsers = await database.db("guitar-practice-suite").collection("users").insertOne({email: email, password: hashedPassword});
insertUserToExercises = await database.db("guitar-practice-suite").collection("exercises").insertOne({email: email});
insertUserToCustomExercises = await database.db("guitar-practice-suite").collection("custom-exercises").insertOne({email: email, exercises: []});
} catch(err) {
log("ERROR while inserting user into database at: createPracticeProfile");
console.log(err);
client.close();
return false;
} finally {
client.close();
return insertUserToUsers && insertUserToExercises && insertUserToCustomExercises;
}
}
I've found the solution to the problem, but I'm not sure I understand the reasoning.
The client.close() in the finally block of the validateUniqueUser function. It was closing the connection before the connection in the createPracticeProfile function was finished inserting the user.
When that line is taken out, the function works.
The issue is client variable needs to be reinstantiated again,
const client = new MongoClient(uri, { useUnifiedTopology: true}, { useNewUrlParser: true }, { connectTimeoutMS: 30000 }, { keepAlive: 1});
Try putting this in start of createPracticeProfile, validateUniqueUser and other functions
I was getting the error
MongoError: Topology is closed
because of the authentication problem
MongoEror: Authentication failed
In my case, the problem was with the password of my database. My password only contained numerical digits.
I changed the password to all characters and both the errors were solved.
Configure your client connection like below example
var MongoClient = require('mongodb').MongoClient;
var Server = require('mongodb').Server;
var mongoClient = new MongoClient(new Server('localhost', 27017));
mongoClient.open(function(err, mongoClient) {
var db1 = mongoClient.db("mydb");
mongoClient.close();
});
In my case - connecting to AtlasDB using the MongoClient - I had to whitelist the IP i was accessing the cluster from
I think your mongodb service is stopped, to start it
Task Manager -> Services -> Mongodb -> RightClick -> Start
My code has been working fine for a long time and hasn't thrown this error before: MongoError: Topology is closed.
But due to the fact that my laptop was turned on for a long time and I was simultaneously developing other projects on it, while the main one was running in the terminal, mongo most likely did not close one of the connections to the database and opened another in parallel, creating some kind of collision.
In general, in my case, the usual restart of the computer helped and a similar error did not occur again.
Related
I'm just starting to use Mongodb without mongoose (to get away from the schemas), and wanted to create a simple module with various exported functions to use in the rest of my app. I've pasted the code below.
The problem I'm having is that the databasesList.databases comes back as undefined, and I'm not sure why. There should be 2 databases on my cluster, and one collection in each database.
As a tangential question, I thought maybe I would check the collections instead (now commented out), but though I found this page (https://docs.mongodb.com/manual/reference/method/db.getCollectionNames/) the function getCollectionNames seems not to exist. Now I'm wondering if I'm using the wrong documentation and that is why my databases are coming back undefined.
const client = new MongoClient(uri)
const connection = client.connect( function (err, database) {
if (err) throw err;
else if (!database) console.log('Unknown error connecting to database');
else {
console.log('Connected to MongoDB database server');
}
});
module.exports = {
getDatabaseList: function() {
console.log('start ' + client);
databasesList = client.db().admin().listDatabases();
//collectionList = client.db().getCollectionNames();
//console.log("Collections: " + collectionList);
console.log("Databases: " + databasesList.databases);
//databasesList.databases.forEach(db => console.log(` - ${db.name}`));
}
}```
your code is correct Just need to change few things.
module.exports = {
getDatabaseList: async function() {
console.log('start ' + client);
databasesList = await client.db().admin().listDatabases();
//collectionList = await client.db().getCollectionNames();
//console.log("Collections: " + collectionList);
console.log("Databases: " + databasesList.databases);
databasesList.databases.forEach(db => console.log(` - ${db.name}`));
}
}
You have to declare async function and use await also.
The async and await keywords enable asynchronous, promise-based behaviour to be written in a cleaner style, avoiding the need to explicitly configure promise chains.
You can use this modular approach to build your database access code:
index.js: Run your database application code, like list database names, collection names and read from a collection.
const connect = require('./database');
const dbFunctions = require('./dbFunctions');
const start = async function() {
const connection = await connect();
console.log('Connected...');
const dbNames = await dbFunctions.getDbNames(connection);
console.log(await dbNames.databases.map(e => e.name));
const colls = await dbFunctions.getCollNames(connection, 'test');
console.log(await colls.map(e => e.name));
console.log(await dbFunctions.getDocs(connection, 'test', 'test'));
};
start();
database.js:: Create a connection object. This connection is used for all your database access code. In general, a single connection creates a connection pool and this can be used throughout a small application
const { MongoClient } = require('mongodb');
const url = 'mongodb://localhost:27017/';
const opts = { useUnifiedTopology: true };
async function connect() {
console.log('Connecting to db server...');
return await MongoClient.connect(url, opts );
}
module.exports = connect;
dbFunctions.js:: Various functions to access database details, collection details and query a specific collection.
module.exports = {
// return list of database names
getDbNames: async function(conn) {
return await conn.db().admin().listDatabases( { nameOnly: true } );
},
// return collections list as an array for a given database
getCollNames: async function(conn, db) {
return await conn.db(db).listCollections().toArray();
},
// return documents as an array for a given database and collection
getDocs: async function(conn, db, coll) {
return await conn.db(db).collection(coll).find().toArray();
}
}
I need to set for each connection in a connection pool default session configurations like TIME_ZONE, LNS_DATE_LANGUAGE and others for Oracle database. Knex.js documentation provide a snippet demonstrated below but for PostgreSQL:
var knex = require('knex')({
client: 'pg',
connection: {hos: ''},
pool: {
afterCreate: function (conn, done) {
// in this example we use pg driver's connection API
conn.query('SET timezone="UTC";', function (err) {
if (err) {
// first query failed, return error and don't try to make next query
done(err, conn);
} else {
// do the second query...
conn.query('SELECT set_limit(0.01);', function (err) {
// if err is not falsy, connection is discarded from pool
// if connection aquire was triggered by a query the error is passed to query promise
done(err, conn);
});
}
});
}
}
});
The oracledb way of doing that is defining a callback for the sessionCallback attribute:
async function init() {
try {
await oracledb.createPool({
user: 'USER',
password: 'PWD',
connectString: 'HOST:1521/SERVICE',
sessionCallback: initSession //HERE MY DEFAULT SETTINGS
});
} catch (e) {
console.error(e);
}
}
async function initSession(connection, requestedTag, cb) {
try {
await connection.execute(`
ALTER SESSION SET
TIME_ZONE='UTC'
NLS_LANGUAGE = 'BRAZILIAN PORTUGUESE'
NLS_TERRITORY = 'BRAZIL'
NLS_CURRENCY = 'R$'
NLS_ISO_CURRENCY = 'BRAZIL'
NLS_NUMERIC_CHARACTERS = ',.'
NLS_CALENDAR = 'GREGORIAN'
NLS_DATE_FORMAT = 'DD/MM/YYYY'
NLS_DATE_LANGUAGE = 'BRAZILIAN PORTUGUESE'
NLS_SORT = 'WEST_EUROPEAN'
NLS_TIMESTAMP_FORMAT = 'DD/MM/YYYY HH24:MI:SS'
NLS_DUAL_CURRENCY = 'R$'
`);
} catch (e) {
console.error(e);
cb(e);
} finally {
if (connection) {
await connection.close();
}
}
}
I would like to be able of translating this snippet above for a Knex.js implementation. How to do that?
Does your KNEX use Oracle DB's connection pool?? https://github.com/tgriesser/knex/issues/2665 show it as a future project.
Try the pg snippet you posted, buyt using the ALTER SESSION from your other example.
8 out of ten times everything connects well. That said, I sometimes get a MongoClient must be connected before calling MongoClient.prototype.db error. How should I change my code so it works reliably (100%)?
I tried a code snippet from one of the creators of the Now Zeit platform.
My handler
const { send } = require('micro');
const { handleErrors } = require('../../../lib/errors');
const cors = require('../../../lib/cors')();
const qs = require('micro-query');
const mongo = require('../../../lib/mongo');
const { ObjectId } = require('mongodb');
const handler = async (req, res) => {
let { limit = 5 } = qs(req);
limit = parseInt(limit);
limit = limit > 10 ? 10 : limit;
const db = await mongo();
const games = await db
.collection('games_v3')
.aggregate([
{
$match: {
removed: { $ne: true }
}
},
{ $sample: { size: limit } }
])
.toArray();
send(res, 200, games);
};
module.exports = handleErrors(cors(handler));
My mongo script that reuses the connection in case the lambda is still warm:
// Based on: https://spectrum.chat/zeit/now/now-2-0-connect-to-database-on-every-function-invocation~e25b9e64-6271-4e15-822a-ddde047fa43d?m=MTU0NDkxODA3NDExMg==
const MongoClient = require('mongodb').MongoClient;
if (!process.env.MONGODB_URI) {
throw new Error('Missing env MONGODB_URI');
}
let client = null;
module.exports = function getDb(fn) {
if (client && !client.isConnected) {
client = null;
console.log('[mongo] client discard');
}
if (client === null) {
client = new MongoClient(process.env.MONGODB_URI, {
useNewUrlParser: true
});
console.log('[mongo] client init');
} else if (client.isConnected) {
console.log('[mongo] client connected, quick return');
return client.db(process.env.MONGO_DB_NAME);
}
return new Promise((resolve, reject) => {
client.connect(err => {
if (err) {
client = null;
console.error('[mongo] client err', err);
return reject(err);
}
console.log('[mongo] connected');
resolve(client.db(process.env.MONGO_DB_NAME));
});
});
};
I need my handler to be 100% reliable.
if (client && !client.isConnected) {
client = null;
console.log('[mongo] client discard');
}
This code can cause problems! Even though you're setting client to null, that client still exists, will continue connecting to mongo, will not be garbage collected, and its callback connection code will still run, but in its callback client will refer to the next client that's created that is not necessarily connected.
A common pattern for this kind of code is to only ever return a single promise from the getDB call:
let clientP = null;
function getDb(fn) {
if (clientP) return clientP;
clientP = new Promise((resolve, reject) => {
client = new MongoClient(process.env.MONGODB_URI, {
useNewUrlParser: true
});
client.connect(err => {
if (err) {
console.error('[mongo] client err', err);
return reject(err);
}
console.log('[mongo] connected');
resolve(client.db(process.env.MONGO_DB_NAME));
});
});
return clientP;
};
I had the same issue. In my case it was caused by calling getDb() before a previous getDb() call had returned. In this case, I believe that 'client.isConnected' returns true, even though it is still connecting.
This was caused by forgetting to put an 'await' before the getDb() call in one location. I tracked down which by outputting a callstack from getDb using:
console.log(new Error().stack);
I don't see the same issue in the sample code in the question, though it could be triggered by another bit of code that isn't shown.
I have written this article talking about serverless, lambda e db connections. There are some good concepts which could help you to find the root cause of your problem. There are also example and use cases of how to mitigate connection pool issues.
Just by looking your code I can tell it is missing this:
context.callbackWaitsForEmptyEventLoop = false;
Serverless: Dynamodb x Mongodb x Aurora serverless
I'm using pg for node to connect to my Postgres server.
Here is what I'm doing:
const Pool = require('pg').Pool;
var config = {
user: "mouser",
password: "my password",
host: "somethingsomething.rds.amazonaws.com",
port: 5432,
database: "mydb",
ssl: true
}
const db = new Pool(config);
console.log("hello world")
let queryString = `CREATE TABLE IF NOT EXISTS favoritememes (id serial PRIMARY KEY, image_url varchar(255), date_favorited TIMESTAMP DEFAULT CURRENT_TIMESTAMP);`
db.query(queryString, function(err, result) {
console.log("Created table");
if (err) console.log("error", err);
});
Currently, I see "hello world" printed to my console, but it never prints "created table" and I also never see an error.
I know that my credentials are correct because I can connect to the server with SQLWorkbench with those same credentials.
I think the issue is that you're not waiting for the pool to finish being created.
Additionally, I suggest using:
Promises for your db functions, it will make you feel better about javascript ;D.
I suggest you use parameterized query functions. They help keep the
code clean and also prevent you needing to deal with variable types
/ heavy escaping.
(BTW, I would put this in another file and export the parameterizedPromise via exports.parameterizedPromise = parameterizedPromise.
Here is a sample of how to use a parameterized query promise:
/*
parameterizedPromise('SELECT * FROM foodtable f WHERE f.flavor = $1 AND f.meal = $2;', ['SPICY', 'BREAKFAST'])
.then(function(result){
console.log(result.rows);
})
.catch(function(error){
console.warn(error);
});
*/
Here is a redacted version of what I had used a while back.
const pg = require('pg');
const DBConfig = {
host: 'xxxxxx',
port: 'xxxxxx',
user: 'xxxxxx',
password: 'xxxxxx',
database: 'xxxxxx',
max: 10, // max number of clients in the pool
ssl: true,
idleTimeoutMillis: 30000 // how long a client is allowed to remain idle before being closed
};
const postgresPool = new pg.Pool(DBConfig, function(){
console.log('connected to postgres database');
// Need to wait for the pool to finish connecting before you can fire off your query.
paramaterizedPromise(`CREATE TABLE IF NOT EXISTS favoritememes (id serial PRIMARY KEY, image_url varchar(255), date_favorited TIMESTAMP DEFAULT CURRENT_TIMESTAMP);`, [])
.then(function(result){
console.log(result);
})
.catch(function(error){
console.warn(error);
});
});
postgresPool.on('error', function (err, client) {
// if an error is encountered by a client while it sits idle in the pool
// the pool itself will emit an error event with both the error and
// the client which emitted the original error
// this is a rare occurrence but can happen if there is a network partition
// between your application and the database, the database restarts, etc.
// and so you might want to handle it and at least log it out
console.error('idle client error', err.message, err.stack);
});
const paramaterizedPromise = function(query, params) {
return new Promise( function(resolve, reject){
postgresPool.connect(function(err, client, done) {
if(err) {
reject('Error Fetching Client From Pool', err);
}
client.query(query, params, function(err, result) {
//call `done()` to release the client back to the pool
done(err);
if (err){
reject(err);
} else {
resolve(result);
}
});
});
});
}
I'm trying to create web services using node.js from an sql server database,in the frontend when i call those 2 webservices simultaneously it throws an error Global connection already exists. Call sql.close() first .
Any Solution ?
var express = require('express');
var router = express.Router();
var sql = require("mssql");
router.get('/Plant/:server/:user/:password/:database', function(req, res, next) {
user = req.params.user;
password = req.params.password;
server = req.params.server;
database = req.params.database;
// config for your database
var config = {
user: user,
password: password,
server: server,
database:database
};
sql.connect(config, function (err) {
// create Request object
var request = new sql.Request();
// query to the database and get the records
request.query("SELECT distinct PlantName FROM MachineryStateTable"
, function (err, recordset) {
if (err) console.log(err)
else {
for(i=0;i<recordset.recordsets.length;i++) {
res.send(recordset.recordsets[i])
}
}
sql.close();
});
});
});
router.get('/Dep/:server/:user/:password/:database/:plantname', function(req, res, next) {
user = req.params.user;
password = req.params.password;
server = req.params.server;
database = req.params.database;
plantname = req.params.plantname;
// config for your database
var config = {
user: user,
password: password,
server: server,
database:database
};
sql.connect(config, function (err) {
// create Request object
var request = new sql.Request();
// query to the database and get the records
request.query("SELECT distinct DepName FROM MachineryStateTable where PlantName= '"+plantname+"'"
, function (err, recordset) {
if (err) console.log(err)
else {
for(i=0;i<recordset.recordsets.length;i++) {
res.send(recordset.recordsets[i])
}
sql.close();
}
});
});
});
module.exports = router;
You have to create a poolConnection
try this:
new sql.ConnectionPool(config).connect().then(pool => {
return pool.request().query("SELECT * FROM MyTable")
}).then(result => {
let rows = result.recordset
res.setHeader('Access-Control-Allow-Origin', '*')
res.status(200).json(rows);
sql.close();
}).catch(err => {
res.status(500).send({ message: `${err}`})
sql.close();
});
From the documentation, close method should be used on the connection, and not on the required module,
So should be used like
var connection = new sql.Connection({
user: '...',
password: '...',
server: 'localhost',
database: '...'
});
connection.close().
Also couple of suggestions,
1. putting res.send in a loop isn't a good idea, You could reply back the entire recordsets or do operations over it, store the resultant in a variable and send that back.
2. Try using promises, instead of callbacks, it would make the flow neater
You must use ConnectionPool.
Next function returns a recordset with my query results.
async function execute2(query) {
return new Promise((resolve, reject) => {
new sql.ConnectionPool(dbConfig).connect().then(pool => {
return pool.request().query(query)
}).then(result => {
resolve(result.recordset);
sql.close();
}).catch(err => {
reject(err)
sql.close();
});
});
}
Works fine in my code!
if this problem still bother you, then change the core api.
go to node_modules\mssql\lib\base.js
at line 1723, add below code before if condition
globalConnection = null
In case someone comes here trying to find out how to use SQL Server pool connection with parameters:
var executeQuery = function(res,query,parameters){
new sql.ConnectionPool(sqlConfig).connect().then(pool =>{
// create request object
var request = new sql.Request(pool);
// Add parameters
parameters.forEach(function(p) {
request.input(p.name, p.sqltype, p.value);
});
// query to the database
request.query(query,function(err,result){
res.send(result);
sql.close();
});
})
}
Don't read their documentation, I don't think it was written by someone that actually uses the library :) Also don't pay any attention to the names of things, a 'ConnectionPool' doesn't seem to actually be a connection pool of any sort. If you try and create more than one connection from a pool, you will get an error. This is the code that I eventually got working:
const sql = require('mssql');
let pool = new sql.ConnectionPool(config); // some object that lets you connect ONCE
let cnn = await pool.connect(); // create single allowed connection on this 'pool'
let result = await cnn.request().query(query);
console.log('result:', result);
cnn.close(); // close your connection
return result;
This code can be run multiple times in parallel and seems to create multiple connections and correctly close them.