Logger callback and Router requests causing SQL Server concurrency issues in Express - node.js

I have a express server that handles queries on a SQL Server server database. For my logs, I am logging all requests that are received into a logs table:
const morgan = require('morgan')
const stream = require('stream')
const DatabaseController = require('./dbcontroller')
const databaseStream = new stream.Writable()
// anything that is logged is passed to this stream... the text is logged
databaseStream.write = async (text) => {
await DatabaseController.addLog({ text })
}
// remove the img binary from the body - which is too large
const logger = morgan( 'tiny', { immediate: true, stream: databaseStream })
module.exports = logger
Additionally, there are some requests that can be made that will also make connections on the database.
router.get('/logs', async (req, res, next) => {
try {
const logdata = await DatabaseController.getLogs()
res.status(200)
res.send({ logdata })
} catch (err) { next(err) }
})
Normal requests are fine. However anytime I make a request that requires connecting to the database (GET/logs). I get the following:
Global connection already exists. Call sql.close() first.
DatabaseController implementation:
const sql = require('mssql')
const config = require('config')
const dconfig = config.database
const addLog = async (options) => {
const { text, isError } = options
try {
let pool = await sql.connect(dconfig)
let request = await pool.request()
request.input('message', sql.Text, text.trim())
if (isError) {
request.input('iserr', sql.Bit, isError)
await request.query('insert into dbo.MApp_Logs (message, is_error) values (#message, #iserr)')
} else {
await request.query('insert into dbo.MApp_Logs (message) values (#message)')
}
} catch (err) {
throw err
} finally {
await sql.close()
}
}
const getLogs = async () => {
try {
let pool = await sql.connect(dconfig)
return await pool.request().query('SELECT * FROM dbo.MApp_Logs')
} catch (err) {
throw err
} finally {
await sql.close()
}
}
The issue is that as is. I can't prevent my code from attempting to make multiple connections. Using a connection pool gives me the following error:
"message": "Already connecting to database! Call close before connecting to different database.",
"code": "EALREADYCONNECTING"
How can I get around this issue? Is it possible to wait for a connection to close or use the existing connection?

Related

Promise.all() throwing error while connecting to database Error: timeout exceeded when used with Node-Postgres

I have Node.js express app with Postgres as a database. I'm using pg for database communication from the app.
This is how my db.service looks like
import { Pool } from 'pg';
const dbConfig = {/*my valid db configuration*/};
const pool = new Pool(dbConfig);
export const connectDB = async () => {
let client;
try {
client = await pool.connect();
} catch (error) {
console.error('error while connecting to database', error);
process.exit(1);
}
return client;
};
I have two queries as below
#1.
export const fetchUser = async (email) => {
const client = await connectDB();
const query = `
SELECT full_name FROM app.users
WHERE email = '${email}'
`;
let result;
try {
result = await client.query(query);
if (result.rowCount) {
return result.rows[0].full_name;
}
} catch (error) {
} finally {
await client.release();
}
return result;
};
#2
export const fetchWallet = async (email) => {
const client = await connectDB();
const query = `
SELECT wallet_money FROM app.user_wallet
WHERE email = '${email}'
`;
let result;
try {
result = await client.query(query);
if (result.rowCount) {
return result.rows[0].wallet_money;
}
} catch (error) {
} finally {
await client.release();
}
return result;
};
Now from one of my controller.js if I call these function separate await, no issues
ctrl.js
const fullName = await fetchUser('some#gmail.com');
const walletMoney = await fetchWallet('some#gmail.com');
No issues this way, however if I merge them into a single promise
const $0= fetchUser('some#gmail.com');
const $1= fetchWallet('some#gmail.com');
const result = await Promise.all([$0, $1]);
this throws the below error
error while connecting to database Error: timeout exceeded when trying
to connect at Error
Please suggest why this error is popping up & how can I get rid of it?
Thanks!
That's happening because you are trying to connect to DB separately for every query.Try to create one connection and use it for all queries!

Node.JS receive data from MongoDB on get request

I'm trying send data from a MongoDB server to the client on a GET request. I'm using an express server and want to send all the documents in a specific collection when a GET request is received.
I know MongoDB is asynchronous, so I query an asynchronous function with a promise. I am able to log to console all the documents in the MongoDB Collection but when I return it the data becomes undefined. I have a database called 'testDB' and I want to return all the documents in a specific collection ('testCollection').
app.get('/getData', (req, res) => {
returnData().then(result => {
console.log(result); //This logs undefined
res.send(result); //This sends undefined
})
});
async function returnData() {
const uri = "mongodb+srv://" + "username" + ":" + "password" + "#" + "connection url" + "/?retryWrites=true&w=majority";
//Connect to the database and return all documents in the collection
const client = new MongoClient(uri, {
useNewUrlParser: true,
useUnifiedTopology: true
});
try {
await client.connect();
const database = client.db('testDB');
const collection = database.collection('testCollection');
const query = {};
const options = {};
const cursor = collection.find(query, options);
await cursor.toArray().then((docs) => {
console.log(docs); // <- This works and logs all the data to console
return docs;
});
} catch (e) {
console.error(e);
} finally {
await client.close();
}
}
EDIT:
I tried this and it returns undefined.
try {
await client.connect();
const database = client.db('testDB');
const collection = database.collection('testCollection');
const query = {};
const options = {};
const cursor = collection.find(query, options);
await cursor.toArray().then((docs) => {
return cursor.toArray();
});
} catch (e) {
console.error(e);
} finally {
await client.close();
}
I tried this and I get [MongoPoolClosedError]: Attempted to check out a connection from closed connection pool
try {
await client.connect();
const database = client.db('testDB');
const collection = database.collection('testCollection');
const query = {};
const options = {};
const cursor = collection.find(query, options);
return cursor.toArray();
} catch (e) {
console.error(e);
} finally {
await client.close();
}
Don't open and close the mongo client on each GET. Connect once and export that connection to your other modules. In terms of the Express request/response. You hand it off something like below.
app.get('/getData', myAsyncFunction)
async function index (req, res) {
let r = await db.collection("testDB").find(query).toArray();
return res.send(r);
}

Express: res.send() returns an empty object

I have a setup that uses a MySQL server node and express, I have created an update statement on a SQL file and got it to work with the node via patch request on the postman.
It seems to work and it updates the server, however, when I try to get the recently updated object or even a success message, I get an empty object or nothing, even though the data has been updated.
Can anyone help me get some sort of response on postman?
update.sql
UPDATE [dbo].[RetailerThemes]
SET [Name] = #name,
[PrimaryColourPalette_main] = #primaryColourPalette_main
WHERE [UniqueThemedPageName] = #UniqueThemedPageName
router.js
router.patch("/update", internal, async (req, res) => {
try {
//do the update
const updateRetailer =
await retailerController.updateRetailerConfigByName(req, res)
console.log(`Update Retailer Routes: ${updateRetailer}`)
res.status(200).send(
{updateRetailer}
);
} catch (error) {
console.log(error);
}
});
controller.js
const updateRetailerConfigByName = async (req, res) => {
try {
// Credentials from Request
let retailername = req.retailername;
// Data from Repository
const thisRetailerConfig = await retailerRep.updateRetailerConfigDetails(
retailername
);
console.log( `thisRetailerConfig: ${thisRetailerConfig}`)
} catch (error) {
console.log(error)
}
};
repo.js
async function updateRetailerConfigDetails(retailername) {
try {
//RetailerTheme
const sqlcommanda = await tsql.loadSql(
"tsql",
//"GetRetailerThemeByThemedPageName.sql"
"UpdateRetailer.sql"
);
let pool = await sql.connect(sqlConfig);
const themes = await pool
.request()
.input("name", sql.VarChar(150), "b") // change 80700 to variable
.input("UniqueThemedPageName", sql.VarChar(150), retailername)
.input("primaryColourPalette_main", sql.VarChar(9), "#c00")
.query(sqlcommanda)
;
if (themes.rowsAffected != 1) {
console.log("Retailer not found for ", retailername, sqlcommanda);
return { isFound: false };
}
const theme = themes.recordset[0];
console.log(`The Theme: ${theme}`)
return theme;
} catch (error) {
console.log(error.message);
return {};
}
}
Here is a screenshot of what I get on the postman, the update works but the response is an empty object.

Nextjs MongoDB template: clientPromise is not a function

I am using Next.js's MongoDB example template and it comes with a MongoDB util function:
import { MongoClient } from 'mongodb';
const uri = process.env.MONGODB_URI;
const options = {};
let client;
let clientPromise;
if (!process.env.MONGODB_URI) {
throw new Error('Please add your Mongo URI to .env.local');
}
if (process.env.NODE_ENV === 'development') {
if (!global._mongoClientPromise) {
client = new MongoClient(uri, options);
global._mongoClientPromise = client.connect();
}
clientPromise = global._mongoClientPromise;
} else {
client = new MongoClient(uri, options);
clientPromise = client.connect();
}
export default clientPromise;
Here is how I am using it and I am certain that I am using this incorrectly.
import { clientPromise } from '../../lib/mongodb';
export default async (req, res) => {
try {
const db = await clientPromise();
const users = await db.collection('users').find({}).limit(20).toArray();
res.json(users);
} catch (e) {
console.error(e);
res.json({ error: 'Not connected!' });
}
};
The error is "TypeError: (0 , lib_mongodb__WEBPACK_IMPORTED_MODULE_0_.clientPromise) is not a function"
You're exporting clientPromise, which is client.connect(). At this point you already "triggered" the connection function. now all you have to do is wait on that promise,
So instead of
const db = await clientPromise();
You should do:
const connection = await clientPromise; // this is a client connection not a db
const db = connection.db();

koa2+koa-router+mysql keep returning 'Not Found'

Background
I am using koa2 with some middlewares to build a basic api framework. But when I use "ctx.body" to send response in my router, the client side always receive "Not Found"
My code
./app.js
const Koa = require('koa');
const app = new Koa();
const config = require('./config');
//Middlewares
const loggerAsync = require('./middleware/logger-async')
const bodyParser = require('koa-bodyparser')
const jsonp = require('koa-jsonp')
app.use(loggerAsync())
app.use(bodyParser())
app.use(jsonp());
//Router
const gateway = require('./router/gateway')
app.use(gateway.routes(), gateway.allowedMethods());
app.use(async(ctx, next) => {
await next();
ctx.response.body = {
success: false,
code: config.code_system,
message: 'wrong path'
}
});
app.listen(3000);
./router/gateway.js
/**
* Created by Administrator on 2017/4/11.
*/
const Router = require('koa-router');
const gateway = new Router();
const df = require('../db/data-fetcher');
const config = require('../config');
const moment = require('moment');
const log4js = require('log4js');
// log4js.configure({
// appenders: { cheese: { type: 'file', filename: 'cheese.log' } },
// categories: { default: { appenders: ['cheese'], level: 'error' } }
// });
const logger = log4js.getLogger('cheese');
logger.setLevel('ERROR');
gateway.get('/gateway', async(ctx, next) => {
let time = ctx.query.time;
if (!time) {
ctx.body = {
success: false,
code: config.code_system,
message: 'Please input running times'
}
} else {
try {
let r = await df(`insert into gateway (g_time, g_result, g_date) values (${time}, '',now())`);
return ctx.body = {
success: true,
code: config.code_success
}
} catch (error) {
logger.error(error.message);
}
}
});
module.exports = gateway;
Then a db wrapper(mysql)
./db/async-db.js
const mysql = require('mysql');
const config = require('../config');
const pool = mysql.createPool({
host: config.database.HOST,
user: config.database.USERNAME,
password: config.database.PASSWORD,
database: config.database.DATABASE
})
let query = (sql, values) => {
return new Promise((resolve, reject) => {
pool.getConnection(function (err, connection) {
if (err) {
reject(err)
} else {
connection.query(sql, values, (err, rows) => {
if (err) {
reject(err)
} else {
resolve(rows)
}
connection.release()
})
}
})
})
}
module.exports = query
./db/data-fetcher.js
const query = require('./async-db')
async function performQuery(sql) {
let dataList = await query(sql)
return dataList
}
module.exports = performQuery;
My running result
When I launch server on port 3000 then accesss via http://localhost:3000/gateway?time=5, it always returns "Not found". But as I can see I have already used
return ctx.body = {
success: true,
code: config.code_success
}
to send response. I debugged and found that the database processing was done well, the new data was inserted well.
when I remove that db inserting line, it works well and returns success info.
let r = await df(`insert into gateway (g_time, g_result, g_date) values (${time}, '',now())`);
Is there anything wrong?
Thanks a lot!
Update 2017/04/27
Now I have found the problem. It's due to my custom middleware
const loggerAsync = require('./middleware/logger-async')
Code are like following -
function log( ctx ) {
console.log( ctx.method, ctx.header.host + ctx.url )
}
module.exports = function () {
return function ( ctx, next ) {
return new Promise( ( resolve, reject ) => {
// 执行中间件的操作
log( ctx )
resolve()
return next()
}).catch(( err ) => {
return next()
})
}
}
I changed it to async/await way then everything is working well.
Could anyone please tell me what's wrong with this middleware?
I guess, your problem is the ./db/data-fetcher.js function. When you are calling
let r = await df(`insert ....`)
your df - function should return a promise.
So try to rewrite your ./db/data-fetcher.js like this (not tested):
const query = require('./async-db')
function performQuery(sql) {
return new Promise((resolve, reject) => {
query(sql).then(
result => {
resolve(result)
}
)
}
}
module.exports = performQuery;
Hope that helps.
correct middleware:
function log( ctx ) {
console.log( ctx.method, ctx.header.host + ctx.url )
}
module.exports = function () {
return function ( ctx, next ) {
log( ctx );
return next()
}
}
reason: when resolve involved; promise chain was completed; response has been sent to client. although middleware remained will involved, but response has gone!
try to understand It seems that if you want to use a common function as middleware, you have to return the next function
nodejs(koa):Can't set headers after they are sent

Resources