Fastify Plugin Performance - node.js

I created a plugin for simple queries with caching and connection pooling. When i respond with that plugin (function), response is slower than before. So i wonder if I got the plugin thing wrong. Is this a correct use or am I making a mistake somewhere?
db.js
const fp = require('fastify-plugin')
const oracledb = require('oracledb');
oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT;
oracledb.autoCommit = true;
module.exports = fp(async function (fastify, opts) {
fastify.decorate('simpleSelectWithCache', async function (key, ttl, sql) {
let cached = await fastify.cache.get(key);
if (cached) {
console.log('Cached:', cached.item);
return cached.item;
} else {
let connection;
try {
connection = await oracledb.getConnection();
const data = await connection.execute(sql);
fastify.cache.set(key, data.rows, ttl);
console.log('Real:', data.rows);
return data.rows;
// oracledb.getPool()._logStats(); // show pool statistics. _enableStats must be true
} catch (error) {
console.error(err);
} finally {
if (connection) await connection.close();
}
}
})
})
api.js
module.exports = async function (fastify, opts) {
fastify.get(
'/cached',
{
schema: {
description: 'Shared Api',
tags: ['Shared'],
},
},
async function (req, reply) {
const data = await fastify.simpleSelectWithCache('shared-cached', 60*1000, 'SELECT id FROM users WHERE id < 50')
reply.send(data);
}
);
};

Is this a correct use or am I making a mistake somewhere?
The connection is a heavy operation and, for every query, a new connection (aka a new socket) is created between your server and DB.
To optimize your plugin you need to create the connection pool at start:
module.exports = fp(async function (fastify, opts) {
await oracledb.createPool({
user: opts.user,
password: opts.password,
connectString: opts.connectString
})
fastify.decorate('simpleSelectWithCache', async function (key, ttl, sql) {
const cached = await fastify.cache.get(key)
if (cached) {
console.log('Cached:', cached.item)
return cached.item
} else {
let connection
try {
connection = await oracledb.getConnection()
const data = await connection.execute(sql)
fastify.cache.set(key, data.rows, ttl)
console.log('Real:', data.rows)
return data.rows
// oracledb.getPool()._logStats(); // show pool statistics. _enableStats must be true
} catch (error) {
console.error(error)
} finally {
if (connection) await connection.close()
}
}
})
fastify.addHook('onClose', (instance, done) => {
oracledb.getPool().close(10)
.then(done)
.catch(done)
})
})
// then register your plugin
fastify.register(myOraclePlugin, {
user: 'ora'
password: '1234',
connectString: 'foo'
})

Related

I get undefined value when I make a request to the MongoDB database using the mongodb engine in Nodejs

I am using ExpressJS and MongoDB to create a blog for myself. I have created a mini library with the mongodb module to request the MongoDB database.
Here is the library:
'use strict'
const { MongoClient, ObjectId } = require('mongodb')
const { config } = require('../config')
const USER = encodeURIComponent(config.mongodb.user)
const PASS = encodeURIComponent(config.mongodb.pass)
const NAME = config.mongodb.name
const HOST = config.mongodb.host
const URL = `mongodb+srv://${USER}:${PASS}#${HOST}/${NAME}?retryWrites=true&w=majority`
const OPTIONS = {
useNewUrlParser: true,
useUnifiedTopology: true
}
class MongoLib {
constructor () {
this.client = new MongoClient(URL, OPTIONS)
this.name = NAME
}
connect () {
if (!MongoLib.connection) {
MongoLib.connection = new Promise((resolve, reject) => {
this.client.connect(err => {
if (err) reject(err)
console.log('Connected successfully to MongoDB.')
resolve(this.client.db(this.name))
})
})
}
return MongoLib.connection
}
getAll (collection, query) {
return this.connect().then(db => {
return db.collection(collection).find({ query }).toArray()
})
}
get (collection, id) {
return this.connect().then(db => {
return db.collection(collection).findOne({ _id: ObjectId(id) })
})
}
create (collection, data) {
return this.connect().then(db => {
return db.collection(collection).insertOne(data)
}).then(result => result.insertedId)
}
update (collection, id, data) {
return this.connect().then(db => {
return db.collection(collection).updateOne({ _id: ObjectId(id) }, { $set: data }, { upsert: true })
}).then(result => result.upsertedId || id)
}
delete (collection, id) {
return this.connect().then(db => {
return db.collection(collection).deleteOne({ _id: ObjectId(id) })
}).then(() => id)
}
}
module.exports = MongoLib
The database is connecting correctly because I have a seed that injects data into the database using the create method of the library that you just saw.
In the service layer, I create a class with a method called getUser, which will call the getAll method of the MongoDB library, to which we pass a query so that it looks for the user.
'use strict'
const MongoLib = require('../lib/mongo')
const bcrypt = require('bcrypt')
class UsersService {
constructor () {
this.collection = 'users'
this.mongoDB = new MongoLib()
}
async getUser ({ email }) {
// { email } is getted by basic authentication as a "username" to login
// I am receiving this data perfectly
const [user] = await this.mongoDB.getAll(this.collection, { email })
// But the problem start here, the value of user is undefined
return user
}
async createUser ({ user }) {
const { name, email, password } = user
const hashedPassword = await bcrypt.hash(password, 10)
const createUserId = await this.mongoDB.create(this.collection, {
name,
email,
password: hashedPassword
})
return createUserId
}
}
module.exports = UsersService
The problem here is that the user value is undefined. I don't understand why it causes conflict. I'm using async-await to wait for the database request to finish, and the data is in the database correctly.
Does anyone have an idea about this error? If more information needs it, please let me know.
Suspect your query is wrong, you are sending { { email: email } } to mongodb
getAll (collection, query) {
return this.connect().then(db => {
return db.collection(collection).find(query).toArray()
})
}

Why is my graphql apollo resolver not being called?

I'm pretty new to graphql (and nodejs as well). I'm following a Udemy course on Apollo and mongo which has been going well mostly. However I can't get one of the resolvers to be called. Another resolver is working fine, and they appear to use the same layout. Also, the context is being called before the resolver that is not being called, so I know it's at least getting that far.
Here is the root server.js with the working context:
const resolvers = require('./resolvers');
...
const apolloServer = new ApolloServer({
typeDefs,
resolvers,
context: async ({ req }) => {
await verifyUser(req);
console.log("=== context ran, user email : ", req.email) ;
return {
email: req.email,
loggedInUserId: req.loggedInUserId
}
}
});
resolvers are modularized, and combined in a /resolvers/index.js, here:
const { GraphQLDateTime } = require('graphql-iso-date')
const userResolver = require('./user');
const taskResolver = require('./task');
const customDateScalarResolver = {
Date: GraphQLDateTime
}
module.exports = [
userResolver,
taskResolver,
customDateScalarResolver
]
and here is the tasks resolver, located at /resolvers/task.js, which is the one not being called:
const uuid = require('uuid')
const { combineResolvers } = require('graphql-resolvers');
const { users, tasks } = require('../constants');
const Task = require('../database/models/task');
const User = require('../database/models/user');
const { isAuthenticated, isTaskOwner } = require('./middleware');
module.exports = {
Query: {
tasks: async ( _, __, { loggedInUserId }) => {
console.log("tasks query, loggedInUserId : ", loggedInUserId);
try {
const tasks = await Task.find( { user: loggedInUserId });
return tasks;
} catch (error) {
console.log(error);
throw error;
}
},
task: async ( parent, { id }, ) => {
console.log("taskbyId query, id : ", id);
// tasks.find(task => task.id == args.id);
try {
const task = await Task.findById(id);
console.log("taskById query, found task? : ", task);
return task;
} catch (error) {
console.log(error);
throw error;
}
},
},
Mutation: {
// createTask: combineResolvers(isAuthenticated, async (_, { input }, { email }) => {
createTask: async (_, { input }, { email }) => {
try {
console.log("creating task, email : ", email);
const user = await User.findOne({ email });
const task = new Task({ ...input, user: user.id });
const result = await task.save();
user.tasks.push(result.id);
await user.save();
return result;
} catch (error) {
console.log(error);
throw error;
}
}
// )
},
Task: {
user: async ( parent ) => {
console.log("in task.user field resolver");
try {
const user = await User.findById(parent.user);
return user;
} catch (error) {
console.log(error);
throw error;
}
}
},
}
When I run the tasks query, the console.log from the context setup function logs 3 times, but does NOT log the console.log line from the tasks resolver. It also appears to not return at all. I'm just using the default graphiql web client. The verifyUser() does find a return a user, so I know the db connection is working fine as well.
mergeResolvers should be used to merge resolvers.
It's designed to merge different [entities] object [/structured] resolvers before use [as one tree structured] in server [config].
F.e. it merges/combines respectively [by type] Query resolvers from users resolver with tasks Query resolvers ... and Mutation resolvers from users resolver with tasks Mutation resolvers.

Bulk update to Postgres with node js performance issue

I'm facing performance issue while trying to do bulk update in PostgresDB. It's taking more than 180 seconds to update around 23000 records. PFB the code. I'm using pg-promise library. Is there anything I could do to improve the performance?
const pgp = require('pg-promise')();
const postgresDBConfig = {
host: Config.postgresDBHost,
port: Config.postgresDBPort,
database: Constants.postgresDBName,
user: Config.postgresDBUser,
password: 'pswd'
};
export async function getTransactionDetails(): Promise<any> {
return new Promise<any>(async function (resolve, reject) {
try {
let db = pgp(postgresDBConfig);
db.connect();
let query = "SELECT * FROM table_name";
db.any(query)
.then(data => {
console.log("Executed successfully::");
resolve(data);
})
.catch(error => {
console.log('ERROR:', error);
})
} catch (error) {
log.error("Error::" + error);
throw error;
}
});
}
export async function updateStatus(result: any, status: string) {
try {
let db = pgp(postgresDBConfig);
//db.connect();
let updateData = [];
_.forEach(result, function (row) {
let updateInfo = {};
updateInfo["sessionid"] = row.sessionid;
updateInfo["status"] = status;
updateData.push(updateInfo);
});
console.log("updateData::" + updateData.length);
const tableName = new pgp.helpers.TableName('table_name', 'schema_name');
let columnset = new pgp.helpers.ColumnSet(['?sessionid', 'status'], { table: tableName });
let update = pgp.helpers.update(updateData, columnset);
db.none(update).then(() => {
console.log("Updated successfully");
})
.catch(error => {
console.log("Error updating the status" + error);
});
}
catch (error) {
log.error("Error in function updateStatus::" + error);
throw error;
}
}
The code exhibits problems all over the place
You should initialize the database object only once
You should not use db.connect() at all, which you also use incorrectly for the async code
You again incorrectly use async block, skipping await, so it doesn't execute correctly.
You do not append any UPDATE logic clause, so it is updating everything all over again, unconditionally, which may be resulting in a delayed mess that you're in.
Here's an improved example, though it may need some more work from your side...
const pgp = require('pg-promise')();
const postgresDBConfig = {
host: Config.postgresDBHost,
port: Config.postgresDBPort,
database: Constants.postgresDBName,
user: Config.postgresDBUser,
password: 'pswd'
};
const db = pgp(postgresDBConfig);
const tableName = new pgp.helpers.TableName('table_name', 'schema_name');
const columnSet = new pgp.helpers.ColumnSet(['?sessionid', 'status'], {table: tableName});
export async function getTransactionDetails(): Promise<any> {
try {
const res = await db.any('SELECT * FROM table_name');
console.log('Executed successfully::');
return res;
} catch (error) {
console.log('ERROR:', error);
throw error;
}
}
export async function updateStatus(result: any, status: string) {
try {
let updateData = [];
_.forEach(result, row => {
let updateInfo = {};
updateInfo["sessionid"] = row.sessionid;
updateInfo["status"] = status;
updateData.push(updateInfo);
});
console.log('updateData::', updateData.length);
const update = pgp.helpers.update(updateData, columnSet) +
' WHERE v.sessionid = t.sessionid';
await db.none(update);
console.log('Updated successfully');
}
catch (error) {
console.log('Error in function updateStatus:', error);
throw error;
}
}

Why does async/await version of my transactional mssql app fail but callbacks version works?

To illuminate the problem I'm having getting a nodejs/mssql application working, I've attempted to code two functionally equivalent versions of a simple (prepared) INSERT statement wrapped in a transaction.
The callbacks version works - inserts a row into my Sql Server db.
The async / await version throws an error -
TransactionError: Can't commit transaction. There is a request in progress.
I have tried many variations of the failing version (statement reordering where plausible), but the version included below is the version that most closely mimics the logic of the working, callbacks version.
Thank you!
var sql = require('mssql'); // mssql: 4.1.0; tedious: 2.2.4; node: v8.4.0
var cfg = {
"db": "sqlserver",
"domain": "XXXXXX",
"user": "cseelig",
"password": "xxxxxx",
"server": "xxxxxx.xxxxxx.xxxxxx.xxxxxx",
"port": 1433,
"stream": false,
"options": {
"trustedConnection": true
},
"requestTimeout": 900000,
"connectionTimeout": 30000,
"pool": {
"max": 3,
"min": 0,
"idleTimeoutMillis": 30000
}
};
var statement = "insert into wng_dw.dbo.D_LIB_Google_Search_Query (query, LastUpdateDate) values (#query, GetDate())";
// I only run one or the other -
main1("12347"); // fails
main2("98765:); // works
async function main1(val) {
try {
const conn = await new sql.connect(cfg);
const transaction = new sql.Transaction();
await transaction.begin();
const ps = new sql.PreparedStatement(transaction);
ps.input('query', sql.VarChar(200));
await ps.prepare(statement);
await ps.execute( {"query": val} );
await ps.unprepare();
await transaction.commit();
sql.close;
} catch(err){
console.log("Error: " + err);
};
process.exit(0);
}
async function main2(val) {
sql.connect(cfg, err => {
const transaction = new sql.Transaction();
transaction.begin(err => {
const ps = new sql.PreparedStatement(transaction);
ps.input('query', sql.VarChar(200));
ps.prepare(statement, err => {
ps.execute( {"query": val}, (err, result) => {
ps.unprepare(err => {
transaction.commit(err => {
sql.close();
});
});
});
});
});
});
}
The transaction.begin does not return a Promise. You could simply promisfy it. Something like the following:
await new Promise(resolve => transaction.begin(resolve));
const request = new sql.Request(transaction);
//...
await transaction.commit();
After the commit and rollback, the "request" object could not be used anymore. Otherwise it will show the error regarding the transaction didn't begin ....
Hope this help.
Before you can commit or rollback a transaction, all statements have to be unprepared.
You have to await the unprepare statement too, otherwise the request is still in progress and the execute promise hasn't resolved yet.
Use a a little wrapper to make things easy:
import * as dotenv from 'dotenv'
import mssql from 'mssql'
dotenv.config()
const sqlServerConfig = {
server: process.env.SQL_SERVER,
user: process.env.QS_USER,
password: process.env.QS_PASS,
options: { enableArithAbort: false },
}
let pool: mssql.ConnectionPool
let transaction: mssql.Transaction
const statements: mssql.PreparedStatement[] = []
export const connect = async (): Promise<void> => {
pool = new mssql.ConnectionPool({ ...sqlServerConfig, database: process.env.DATABASE })
await pool.connect()
}
export const disconnect = async (): Promise<void> => {
if (typeof pool == 'undefined') return
if (pool.connected) await pool.close()
}
export const begin = async (): Promise<void> => {
transaction = new mssql.Transaction(pool)
await transaction.begin()
}
export const unprepare = async (statement: mssql.PreparedStatement): Promise<void> => {
if (typeof statement == 'undefined') return
if (statement.prepared) await statement.unprepare()
}
export const commit = async (): Promise<void> => {
await transaction.commit()
}
export const rollback = async (): Promise<void> => {
for (const statement of statements) {
await unprepare(statement)
}
if (typeof transaction == 'undefined') return
await transaction.rollback()
}
export const createStatement = (): mssql.PreparedStatement => {
const statement = new mssql.PreparedStatement(transaction)
statements.push(statement)
return statement
}
Usage:
try {
await connect()
await begin()
const myStatement = createStatement()
..... bind parameters
..... prepare statement
for ( ..... ) {
await myStatement.execute( ..... )
}
await unprepare(myStatement)
await commit()
await disconnect()
exit(0)
}
catch(e) {
log.error(e)
await rollback()
await disconnect()
exit(1)
}
You create a prepared statement with createStatement(). createStatement keeps track of the statements so in case you rollback they will be unprepared for you when you call rollback.

koa2+koa-router+mysql keep returning 'Not Found'

Background
I am using koa2 with some middlewares to build a basic api framework. But when I use "ctx.body" to send response in my router, the client side always receive "Not Found"
My code
./app.js
const Koa = require('koa');
const app = new Koa();
const config = require('./config');
//Middlewares
const loggerAsync = require('./middleware/logger-async')
const bodyParser = require('koa-bodyparser')
const jsonp = require('koa-jsonp')
app.use(loggerAsync())
app.use(bodyParser())
app.use(jsonp());
//Router
const gateway = require('./router/gateway')
app.use(gateway.routes(), gateway.allowedMethods());
app.use(async(ctx, next) => {
await next();
ctx.response.body = {
success: false,
code: config.code_system,
message: 'wrong path'
}
});
app.listen(3000);
./router/gateway.js
/**
* Created by Administrator on 2017/4/11.
*/
const Router = require('koa-router');
const gateway = new Router();
const df = require('../db/data-fetcher');
const config = require('../config');
const moment = require('moment');
const log4js = require('log4js');
// log4js.configure({
// appenders: { cheese: { type: 'file', filename: 'cheese.log' } },
// categories: { default: { appenders: ['cheese'], level: 'error' } }
// });
const logger = log4js.getLogger('cheese');
logger.setLevel('ERROR');
gateway.get('/gateway', async(ctx, next) => {
let time = ctx.query.time;
if (!time) {
ctx.body = {
success: false,
code: config.code_system,
message: 'Please input running times'
}
} else {
try {
let r = await df(`insert into gateway (g_time, g_result, g_date) values (${time}, '',now())`);
return ctx.body = {
success: true,
code: config.code_success
}
} catch (error) {
logger.error(error.message);
}
}
});
module.exports = gateway;
Then a db wrapper(mysql)
./db/async-db.js
const mysql = require('mysql');
const config = require('../config');
const pool = mysql.createPool({
host: config.database.HOST,
user: config.database.USERNAME,
password: config.database.PASSWORD,
database: config.database.DATABASE
})
let query = (sql, values) => {
return new Promise((resolve, reject) => {
pool.getConnection(function (err, connection) {
if (err) {
reject(err)
} else {
connection.query(sql, values, (err, rows) => {
if (err) {
reject(err)
} else {
resolve(rows)
}
connection.release()
})
}
})
})
}
module.exports = query
./db/data-fetcher.js
const query = require('./async-db')
async function performQuery(sql) {
let dataList = await query(sql)
return dataList
}
module.exports = performQuery;
My running result
When I launch server on port 3000 then accesss via http://localhost:3000/gateway?time=5, it always returns "Not found". But as I can see I have already used
return ctx.body = {
success: true,
code: config.code_success
}
to send response. I debugged and found that the database processing was done well, the new data was inserted well.
when I remove that db inserting line, it works well and returns success info.
let r = await df(`insert into gateway (g_time, g_result, g_date) values (${time}, '',now())`);
Is there anything wrong?
Thanks a lot!
Update 2017/04/27
Now I have found the problem. It's due to my custom middleware
const loggerAsync = require('./middleware/logger-async')
Code are like following -
function log( ctx ) {
console.log( ctx.method, ctx.header.host + ctx.url )
}
module.exports = function () {
return function ( ctx, next ) {
return new Promise( ( resolve, reject ) => {
// 执行中间件的操作
log( ctx )
resolve()
return next()
}).catch(( err ) => {
return next()
})
}
}
I changed it to async/await way then everything is working well.
Could anyone please tell me what's wrong with this middleware?
I guess, your problem is the ./db/data-fetcher.js function. When you are calling
let r = await df(`insert ....`)
your df - function should return a promise.
So try to rewrite your ./db/data-fetcher.js like this (not tested):
const query = require('./async-db')
function performQuery(sql) {
return new Promise((resolve, reject) => {
query(sql).then(
result => {
resolve(result)
}
)
}
}
module.exports = performQuery;
Hope that helps.
correct middleware:
function log( ctx ) {
console.log( ctx.method, ctx.header.host + ctx.url )
}
module.exports = function () {
return function ( ctx, next ) {
log( ctx );
return next()
}
}
reason: when resolve involved; promise chain was completed; response has been sent to client. although middleware remained will involved, but response has gone!
try to understand It seems that if you want to use a common function as middleware, you have to return the next function
nodejs(koa):Can't set headers after they are sent

Resources