nodejs redis Q promises, how to make it work? - node.js

I am trying to get few values from redis, combine them and eventually send. But I just can't make those promises work.
This is the simple get functions from redis
client.get('user:1:id',function(err,data){
// here I have data which contains user ID
});
client.get('user:1:username',function(err,data){
// here I have data which contains username
});
Now I want to get ID and username and send them, but I have no idea how to make that work. I manage to make it work with callbacks but it is very messy result, so then i tried to wrap anonymous functions into Q.fcall and after call .then which looks something like that
client.get('user:1:id',Q.fcall(function(err,data){
return data;
}).then(function(val) {
// do something
}));
but that gives me error for too many arguments been passed and I'm not even sure if that would help me even if it would work.

Q.all([Q.ninvoke(client, 'get', 'user:1:id'),
Q.ninvoke(client, 'get', 'user:1:username')]).then(function (data) {
var id = data[0];
var username = data[1];
// do something with them
});
See https://github.com/kriskowal/q#adapting-node

I use a simple RequireJS module using node-redis and whenjs to create a lifted redis wrapper:
define [
'redis/lib/commands'
'when'
'when/node/function'
], (Commands, When, NodeFn) ->
'use strict'
lift = (redis) ->
wrapped = {}
Commands.map (cmd) ->
wrapped[cmd] = (args...) ->
def = When.defer()
args.push NodeFn.createCallback def.resolver
redis[cmd].apply redis, args
def.promise
wrapped
{lift}
Usage is straightforward:
client = lift redis.createClient()
client.get("hello").then console.log, console.error

Using Promise, Bluebird and node_redis:
import { RedisClient, createClient, ClientOpts } from "redis";
import { promisifyAll, PromisifyAllOptions } from "bluebird";
export module FMC_Redis {
export class Redis {
opt: ClientOpts;
private rc: RedisClient;
private rcPromise: any;
private static _instance: Redis = null;
public static current(_opt?: ClientOpts): Redis {
if (!Redis._instance) {
Redis._instance = new Redis(_opt);
Redis._instance.redisConnect();
}
return Redis._instance;
}
public get client(): RedisClient {
if (!this.rc.connected) throw new Error("There is no connection to Redis DB!");
return this.rc;
}
/******* BLUEBIRD ********/
public get clientAsync(): any {
// promisifyAll functions of redisClient
// creating new redis client object which contains xxxAsync(..) functions.
return this.rcPromise = promisifyAll(this.client);
}
private constructor(_opt?: ClientOpts) {
if (Redis._instance) return;
this.opt = _opt
? _opt
: {
host: "127.0.0.1",
port: 6379,
db: "0"
};
}
public redisConnect(): void {
this.rc = createClient(this.opt);
this.rc
.on("ready", this.onReady)
.on("end", this.onEnd)
.on("error", this.onError);
}
private onReady(): void { console.log("Redis connection was successfully established." + arguments); }
private onEnd(): void { console.warn("Redis connection was closed."); }
private onError(err: any): void { console.error("There is an error: " + err); }
/****** PROMISE *********/
// promise redis test
public getRegularPromise() {
let rc = this.client;
return new Promise(function (res, rej) {
console.warn("> getKeyPromise() ::");
rc.get("cem", function (err, val) {
console.log("DB Response OK.");
// if DB generated error:
if (err) rej(err);
// DB generated result:
else res(val);
});
});
}
/******* ASYNC - AWAIT *******/
// async - await test function
public delay(ms) {
return new Promise<string>((fnResolve, fnReject) => {
setTimeout(fnResolve("> delay(" + ms + ") > successfull result"), ms);
});
}
public async delayTest() {
console.log("\n****** delayTest ")
let a = this.delay(500).then(a => console.log("\t" + a));
let b = await this.delay(400);
console.log("\tb::: " + b);
}
// async - await function
public async getKey(key: string) {
let reply = await this.clientAsync.getAsync("cem");
return reply.toString();
}
}
}
let a = FMC_Redis.Redis.current();
// setTimeout(function () {
// console.warn(a.client.set("cem", "naber"));
// console.warn(a.client.get("cem"));
// console.warn(a.client.keys("cem"));
// }, 1000);
/***** async await test client *****/
a.delayTest();
/** Standart Redis Client test client */
setTimeout(function () {
a.client.get("cem", function (err, val) {
console.log("\n****** Standart Redis Client")
if (err) console.error("\tError: " + err);
else console.log("\tValue ::" + val);
});
}, 100)
/***** Using regular Promise with Redis Client > test client *****/
setTimeout(function () {
a.getRegularPromise().then(function (v) {
console.log("\n***** Regular Promise with Redis Client")
console.log("\t> Then ::" + v);
}).catch(function (e) {
console.error("\t> Catch ::" + e);
});
}, 100);
/***** Using bluebird promisify with Redis Client > test client *****/
setTimeout(function () {
var header = "\n***** bluebird promisify with Redis Client";
a.clientAsync.getAsync("cem").then(result => console.log(header + result)).catch(console.error);
}, 100);

Related

Unable to mock ioredis connection using sinon

I'm trying to create a unit test for the following service, using Sinon.
as you can see the "_createRedisConnection" is called on the constructor, so in the unit test I must mock the Redis connection.
import { inject, injectable } from "inversify";
import { TYPES } from "../../inversify/types";
import { Logger } from "winston";
import { Config } from "../../interfaces/config.interface";
import { BaseService } from "../base.service";
import * as Redis from "ioredis";
import { HttpResponseError } from "../../interfaces/HttpResponseError.interface";
import { BaseResponse } from "../../interfaces/BaseResponse.interface";
#injectable()
export class RedisService extends BaseService {
private _redisClient;
private _isRedisConnected: boolean;
constructor(#inject(TYPES.Logger) private logger: Logger,
#inject(TYPES.Config) private config: Config) {
super(logger, config);
this._isRedisConnected = false;
this._createRedisConnection();
}
public async set(key, value, epu, receivedTtl): Promise<BaseResponse> {
if (this._isRedisConnected) {
const encryptedKey = this.createEncryptedKey(epu, key);
if (!encryptedKey || !value) {
throw new HttpResponseError("General error", "Missing attributes in request body", 422);
}
const ttl = this.limitTtl(receivedTtl);
let response;
if (ttl >= 0) {
await this._redisClient.setex(encryptedKey, ttl, value)
.then(() => {
response = new BaseResponse("success", "Data saved successfully", ttl);
})
.catch((errorMessage: string) => {
throw new HttpResponseError("General error", `Error while saving data. err = ${errorMessage}`, 500);
});
} else {
await this._redisClient.set(encryptedKey, value)
.then(() => {
response = new BaseResponse("success", "Data saved successfully", ttl);
})
.catch((errorMessage: string) => {
throw new HttpResponseError("General error", `Error while saving data. err = ${errorMessage}`, 500);
});
}
return response;
}
throw new HttpResponseError("General error", "Cache is not responding", 503);
}
private _createRedisConnection(): void {
this._redisClient = new Redis({
sentinels: [{ host: this.config.redisConfig.host, port: this.config.redisConfig.port }],
name: "mymaster",
dropBufferSupport: true,
});
this._redisClient.on("connect", () => {
this._isRedisConnected = true;
});
this._redisClient.on("error", (errorMessage: string) => {
this._isRedisConnected = false;
});
}
}
My problem is with mocking the Redis connection. I'm trying stub the 'connect' event, but while debugging it I see that the event never triggered (even not the error event).
import "reflect-metadata";
import { expect } from "chai";
import { Logger } from "winston";
import * as Redis from "ioredis";
import { stub } from "sinon";
import { RedisService } from "./redis.service";
import { config } from "../../config";
class LoggerMock {
public info(str: string) { }
public error(str: string) { }
}
describe("RedisService Service", () => {
const redisStub = stub(Redis.prototype, "connect").returns(Promise.resolve());
const logger = new LoggerMock() as Logger;
const redisService = new RedisService(logger, config);
it("Should success set data", async () => {
const redisClientStub = stub(Redis.prototype, "set").resolves(new Promise((resolve, reject) => { resolve('OK'); }));
const result = await redisService.set("key", "value", "epu", -1);
expect(result.message).to.equals("success");
expect(result.response).to.equals("Data saved successfully");
redisClientStub.restore();
redisStub.restore();
});
});
What is the right way to test this service? why no event is triggered when stubbing this way?
Thanks
This is an example to how to stub ioredis Redis.prototype.connect.
// File test.js
const { expect } = require('chai');
const Redis = require('ioredis');
const sinon = require('sinon');
describe('connection', function () {
it('should emit "connect" when connected', function (done) {
// Create stub on connect.
const stubRedisConnect = sinon.stub(Redis.prototype, 'connect');
stubRedisConnect.callsFake(async function () {
// This will trigger connect event.
this.setStatus('connect');
});
const redis = new Redis();
redis.on('connect', function () {
// Do not forget to restore the stub.
stubRedisConnect.restore();
done();
});
});
});
When I run it on my terminal:
$ npx mocha test.js
connection
✓ should emit "connect" when connected
1 passing (6ms)
If the test stub failed, there will be default timeout error for 2000ms because done not get called.

Fastify Plugin Performance

I created a plugin for simple queries with caching and connection pooling. When i respond with that plugin (function), response is slower than before. So i wonder if I got the plugin thing wrong. Is this a correct use or am I making a mistake somewhere?
db.js
const fp = require('fastify-plugin')
const oracledb = require('oracledb');
oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT;
oracledb.autoCommit = true;
module.exports = fp(async function (fastify, opts) {
fastify.decorate('simpleSelectWithCache', async function (key, ttl, sql) {
let cached = await fastify.cache.get(key);
if (cached) {
console.log('Cached:', cached.item);
return cached.item;
} else {
let connection;
try {
connection = await oracledb.getConnection();
const data = await connection.execute(sql);
fastify.cache.set(key, data.rows, ttl);
console.log('Real:', data.rows);
return data.rows;
// oracledb.getPool()._logStats(); // show pool statistics. _enableStats must be true
} catch (error) {
console.error(err);
} finally {
if (connection) await connection.close();
}
}
})
})
api.js
module.exports = async function (fastify, opts) {
fastify.get(
'/cached',
{
schema: {
description: 'Shared Api',
tags: ['Shared'],
},
},
async function (req, reply) {
const data = await fastify.simpleSelectWithCache('shared-cached', 60*1000, 'SELECT id FROM users WHERE id < 50')
reply.send(data);
}
);
};
Is this a correct use or am I making a mistake somewhere?
The connection is a heavy operation and, for every query, a new connection (aka a new socket) is created between your server and DB.
To optimize your plugin you need to create the connection pool at start:
module.exports = fp(async function (fastify, opts) {
await oracledb.createPool({
user: opts.user,
password: opts.password,
connectString: opts.connectString
})
fastify.decorate('simpleSelectWithCache', async function (key, ttl, sql) {
const cached = await fastify.cache.get(key)
if (cached) {
console.log('Cached:', cached.item)
return cached.item
} else {
let connection
try {
connection = await oracledb.getConnection()
const data = await connection.execute(sql)
fastify.cache.set(key, data.rows, ttl)
console.log('Real:', data.rows)
return data.rows
// oracledb.getPool()._logStats(); // show pool statistics. _enableStats must be true
} catch (error) {
console.error(error)
} finally {
if (connection) await connection.close()
}
}
})
fastify.addHook('onClose', (instance, done) => {
oracledb.getPool().close(10)
.then(done)
.catch(done)
})
})
// then register your plugin
fastify.register(myOraclePlugin, {
user: 'ora'
password: '1234',
connectString: 'foo'
})

MongoDB in NodeJS Class Structure

Is there a way to use MongoDB in a class structure in NodeJS?
I understand you can do CRUD operations on the DB within the connection method like
mongo.connect(url, function(err, client){//do some CRUD operation});
but I was wondering if there was a way to open the connection to the DB, have access to it across the class, then close it when you are done working with the class.
For example:
class MyClass {
constructor(databaseURL) {
this.url = databaseURL;
}
async init() {
//make connection to database
}
async complete_TaskA_onDB() {
//...
}
async complete_TaskB_onDB() {
//...
}
async close_connection() {
//close connection to database
}
}
Edit:
I just came across more information in the Node.JS Mongo docs. Maybe something along the lines of this would work?
//constructor()
this.db = new MongoClient(new Server(dbHost, dbPort));
//init()
this.db.open();
//taskA()
this.db.collection(...).update(...);
//close_connection()
this.db.close();
You can create a class of which will act as a wrapper on any core lib, doing so will give you below advantages:
Wrapping any core module with your own service will allow you to:
Create a reusable service that you could use on multiple components in your app.
Normalize the module's API, and add more methods your app needs and the module doesn’t provide.
Easily replace the DB module you chose with another one (if needed).
I have created that service which I use it in my projects for MongoDB:
var mongoClient = require("mongodb").MongoClient,
db;
function isObject(obj) {
return Object.keys(obj).length > 0 && obj.constructor === Object;
}
class mongoDbClient {
async connect(conn, onSuccess, onFailure){
try {
var connection = await mongoClient.connect(conn.url, { useNewUrlParser: true });
this.db = connection.db(conn.dbName);
logger.info("MongoClient Connection successfull.");
onSuccess();
}
catch(ex) {
logger.error("Error caught,", ex);
onFailure(ex);
}
}
async getNextSequence(coll) {
return await this.db.collection("counters").findOneAndUpdate({
_id: coll
},
{$inc: {seq: 1}},
{projections: {seq: 1},
upsert: true,
returnOriginal: false
}
);
}
async insertDocumentWithIndex(coll, doc) {
try {
if(!isObject(doc)){
throw Error("mongoClient.insertDocumentWithIndex: document is not an object");
return;
}
var index = await this.getNextSequence(coll);
doc.idx = index.value.seq;
return await this.db.collection(coll).insertOne(doc);
}
catch(e) {
logger.error("mongoClient.insertDocumentWithIndex: Error caught,", e);
return Promise.reject(e);
}
}
async findDocFieldsByFilter(coll, query, projection, lmt) {
if(!query){
throw Error("mongoClient.findDocFieldsByFilter: query is not an object");
}
return await this.db.collection(coll).find(query, {
projection: projection || {},
limit: lmt || 0
}).toArray();
}
async findDocByAggregation(coll, query) {
if(!query.length){
throw Error("mongoClient.findDocByAggregation: query is not an object");
}
return this.db.collection(coll).aggregate(query).toArray();
}
async getDocumentCountByQuery(coll, query) {
return this.db.collection(coll).estimatedDocumentCount(query || {})
}
async findOneAndUpdate(coll, query, values, option) {
if(!(isObject(values) && isObject(query))){
throw Error("mongoClient.UpdateDocument: values and query should be an object");
}
return this.db.collection(coll).findOneAndUpdate(query, {$set : values}, option || {})
}
async modifyOneDocument(coll, query, values, option) {
if(!(isObject(values) && isObject(query))){
throw Error("mongoClient.ModifyOneDocument: values, query and option should be an object");
}
return await this.db.collection(coll).updateOne(query, values, option || {})
}
async close() {
return await this.db.close();
}
}
module.exports = {
mongoDbClient: mongoDbClient
}
For my complete lib access you can refer here
Yes, you can do all of this inside a class, but you cannot set a member variable like db after the constructor has been set. You can make it a global variable, but you cannot set the variable.
const MongoClient = require('mongodb').MongoClient;
var database; //global
class DB {
constructor(url, dbName) {
this.url = url;
this.dbName = dbName;
}
connect() {
console.log('connecting to database ' + this.dbName + ' with URL ' + this.url);
return new Promise((resolve, reject) => {
MongoClient.connect(this.url, (err, client) => {
if (err) {
reject(err);
} else {
database = client.db(this.dbName);
resolve(client.db(this.dbName));
}
});
})
}
}

share mqtt client object between files

I connect to MQTT this way:
//mqtt.js
const mqtt = require('mqtt');
var options = {
//needed options
};
var client = mqtt.connect('mqtt://someURL', options);
client.on('connect', () => {
console.log('Connected to MQTT server');
});
I want to export the client object this way:
//mqtt.js
module.exports = client;
So that I can import it in other files and make use of it this way:
//anotherFile.js
const client = require('./mqtt');
client.publish(...)
However, we all know that this will not work! How can I achieve this ?
Update
I tried promise and get a very strange behavior. When I use the promise in the same file (mqtt.js) like the code below, everything is OK:
//mqtt.js
const mqtt = require('mqtt');
var mqttPromise = new Promise(function (resolve, reject) {
var options = {
//needed options
};
var client = mqtt.connect('mqtt://someURL', options);
client.on('connect', () => {
client.subscribe('#', (err) => {
if (!err) {
console.log('Connected to MQTT server');
resolve(client);
} else {
console.log('Error: ' + err);
reject(err);
}
});
});
});
mqttPromise.then(function (client) {
//do sth with client
}, function (err) {
console.log('Error: ' + err);
});
But when I export the promise and use it in another file, like this:
//mqtt.js
//same code to create the promise
module.exports = mqttPromise;
//anotherFile.js
const mqttPromise = require('./mqtt');
mqttPromise.then(function (client) {
//do sth with client
}, function (err) {
console.log('Error: ' + err);
});
I get this error:
TypeError: mqttPromise.then is not a function
You can probably achieve your goal creating 2 files, one for handling mqtt methods and another to manage the connection object.
Here's the file for the mqtt handler:
//mqttHandler.js
const mqtt = require('mqtt');
class MqttHandler {
constructor() {
this.mqttClient = null;
this.host = 'YOUR_HOST';
this.username = 'YOUR_USER';
this.password = 'YOUR_PASSWORD';
}
connect() {
this.mqttClient = mqtt.connect(this.host, {port: 1883});
// Mqtt error calback
this.mqttClient.on('error', (err) => {
console.log(err);
this.mqttClient.end();
});
// Connection callback
this.mqttClient.on('connect', () => {
console.log(`mqtt client connected`);
});
this.mqttClient.on('close', () => {
console.log(`mqtt client disconnected`);
});
}
// // Sends a mqtt message to topic: mytopic
sendMessage(message, topic) {
this.mqttClient.publish(topic, JSON.stringify(message));
}
}
module.exports = MqttHandler;
Now lets use the exported module to create a mqtt client connection on another file:
//mqttClient.js
var mqttHandler = require('./mqttHandler');
var mqttClient = new mqttHandler();
mqttClient.connect();
module.exports = mqttClient;
With this exported module you can now call your client connection object and use the methods created in the mqttHandler.js file in another file :
//main.js
var mqttClient = require('./mqttClient');
mqttClient.sendMessage('<your_topic>','<message>');
Although there may be a better method to perform your task, this one worked pretty well for me...
Hope it helps!
cusMqtt.js
const mqtt = require("mqtt");
function prgMqtt() {
const options = {
port: 1883,
host: "mqtt://xxxxxxx.com",
clientId: "mqttjs_" + Math.random().toString(16).substr(2, 8),
username: "xxxxxx",
password: "xxxxxx",
keepalive: 60,
reconnectPeriod: 1000,
protocolId: "MQIsdp",
protocolVersion: 3,
clean: true,
encoding: "utf8",
};
prgMqtt.client = mqtt.connect("mqtt://xxxxxxxx.com", options);
prgMqtt.client.on("connect", () => {
prgMqtt.client.subscribe("Room/Fan");
console.log("connected MQTT");
});
prgMqtt.client.on("message", (topic, message) => {
console.log("message is " + message);
console.log("topic is " + topic);
// client.end();
});
}
exports.prgMqtt = prgMqtt;
index.js/main program call
const { prgMqtt } = require("./startup/cusMqtt");
prgMqtt();
another .js
const { prgMqtt } = require("../startup/cusMqtt");
router.get("/:id", async (req, res) => {
prgMqtt.client.publish("Room/Reply", "Replied Message");
});

how to connect to mongodb synchronously in nodejs

I want to make use of the promises feature where in I can connect to mongodb synchronously and I can reuse the connection by passing it on to different modules.
Here is something that I came up with
class MongoDB {
constructor(db,collection) {
this.collection = db.collection(collection);
}
find(query, projection) {
if(projection)
return this.collection.find(query, projection);
else
return this.collection.find(query);
}
}
class Crew extends MongoDB {
constructor(db) {
super(db,'crews');
}
validate() {
}
}
I want to setup a connection somewhere in my initial code like the one below and then reuse the connection for different classes, just like how mongoose or monk does but using only the node-mongodb-native package.
MongoClient.connect(url)
.then( (err,dbase) => {
global.DB = dbase;
});
var Crew = new CrewModel(global.DB);
Crew.find({})
.then(function(resp) {
console.log(resp);
});
Right now, the db returns undefined inside the main MongoDB class and am not able to debug this one out through google or the documentation.
Edit: I had assumed that a promise was synchronous but that is not the case.
To reuse the connection I would create a module like this.
module.exports = {
connect: function(dbName, callback ) {
MongoClient.connect(dbName, function(err, db) {
_db = db;
return callback( err );
});
},
getDb: function() {
return _db;
}
};
After that you can connect to the database before starting your application
MongoConnection.connect("mongodb://localhost:27017/myDatabase", function(err){
app.listen(3000, function () {
// you code
});
});
Considering you created the module in a js file you can simply use require to get the databaseConnection
var dbConnection = require("./myMongoConnection.js");
and to get the connection use
var db = MongoConnection.getDb();
Another option using ES6 classes creates a singleton object that you can access repeatedly. It's inspired by #user3134009's answer here.
const EventEmitter = require('events');
const MongoClient = require('mongodb').MongoClient;
const config = require('config');
let _db = null;
class MongoDBConnection extends EventEmitter {
constructor() {
super();
this.emit("dbinit", this);
if (_db == null) {
console.log("Connecting to MongoDB...");
MongoClient.connect(config.dbs.mongo.url, config.dbs.mongo.options,
(err, db) => {
if (err) {
console.error("MongoDB Connection Error", err);
_db = null;
} else {
console.log("Connected to MongoDB", config.dbs.mongo.url);
db.on('close', () => { console.log("MongoDB closed", arguments); _db = null; });
db.on('reconnect', () => { console.log("MongoDB reconnected", arguments); _db = db; });
db.on('timeout', () => { console.log("MongoDB timeout", arguments); });
_db = db;
this.emit('dbconnect', _db);
}
});
}
}
getDB() {
return _db;
}
}
module.exports = new MongoDBConnection();
I have been struggling with this problem for a while, and in particular with setting up and persisting MongoDb connection in AWS lambda functions across invocations.
Thanks to #toszter answer I've finally come up with the following solution:
const mongodb = require('mongodb');
const config = require('./config.json')[env];
const client = mongodb.MongoClient;
const mongodbUri = `mongodb://${config.mongo.user}:${config.mongo.password}#${config.mongo.url}/${config.mongo.database}`;
const options = {
poolSize: 100,
connectTimeoutMS: 120000,
socketTimeoutMS: 1440000
};
// connection object
let _db = null;
class MongoDBConnection {
constructor() {}
// return a promise to the existing connection or the connection function
getDB() {
return (_db ? Promise.resolve(_db) : mConnect());
}
}
module.exports = new MongoDBConnection();
// transforms into a promise Mongo's client.connect
function mConnect() {
return new Promise((resolve, reject) => {
console.log('Connecting to Mongo...');
client.connect(mongodbUri, options, (error, db) => {
if (error) {
_db = null;
return reject(error);
}
else {
console.log('Connected to Mongo...');
_db = db;
resolve(db);
}
});
});
}
To use it in a controller or app.js:
const mongoConfig = require('mongoConfig');
mongoConfig.getDB()
.then(db => db.collection('collection').find({}))
.catch(error => {...});

Resources