I'm new to using the generic-pool. I see that there is a maxWaitingClients setting, but it doesn't specify for how long a request can stay in the queue before it times out. Is there any way for me to specify such a timeout setting?
EDIT: Added my code for how I'm currently using generic-pool:
function createPool() {
const factory = {
create: function() {
return new Promise((resolve, reject) => {
const socket = new net.Socket();
socket.connect({
host: sdkAddress,
port: sdkPort,
});
socket.setKeepAlive(true);
socket.on('connect', () => {
resolve(socket);
});
socket.on('error', error => {
reject(error);
});
socket.on('close', hadError => {
console.log(`socket closed: ${hadError}`);
});
});
},
destroy: function(socket) {
return new Promise((resolve) => {
socket.destroy();
resolve();
});
},
validate: function (socket) {
return new Promise((resolve) => {
if (socket.destroyed || !socket.readable || !socket.writable) {
return resolve(false);
} else {
return resolve(true);
}
});
}
};
return genericPool.createPool(factory, {
max: poolMax,
min: poolMin,
maxWaitingClients: poolQueue,
testOnBorrow: true
});
}
const pool = createPool();
async function processPendingBlocks(ProcessingMap, channelid, configPath) {
setTimeout(async () => {
let nextBlockNumber = fs.readFileSync(configPath, "utf8");
let processBlock;
do {
processBlock = ProcessingMap.get(channelid, nextBlockNumber);
if (processBlock == undefined) {
break;
}
try {
const sock = await pool.acquire();
await blockProcessing.processBlockEvent(channelid, processBlock, sock, configPath, folderLog);
await pool.release(sock);
} catch (error) {
console.error(`Failed to process block: ${error}`);
}
ProcessingMap.remove(channelid, nextBlockNumber);
fs.writeFileSync(configPath, parseInt(nextBlockNumber, 10) + 1);
nextBlockNumber = fs.readFileSync(configPath, "utf8");
} while (true);
processPendingBlocks(ProcessingMap, channelid, configPath);
}, blockProcessInterval)
}
Since you are using pool.acquire(), you would use the option acquireTimeoutMillis to set a timeout for how long .acquire() will wait for an available resource from the pool before timing out.
You would presumably add that option here:
return genericPool.createPool(factory, {
max: poolMax,
min: poolMin,
maxWaitingClients: poolQueue,
testOnBorrow: true,
acquireTimeoutMillis, 3000 // max time to wait for an available resource
});
Related
What I'm trying to do is to emit a event based on the progress of my jobs. It's working in the Gateway(proxy) side, the logs appear, etc, but when I'm consuming the event on the front-end, sometimes it works, sometimes not, and it throws 'ping timeout' error in the console. If I restart the nodejs service a few times it works.
I'm open to ideas of alternative ways to implement this feature.
SocketController
export default class SocketController {
socket: any;
interval: any;
instance: any;
queue: any;
constructor(server) {
// Creating Websocket connection
this.instance = new Server(server, {
cors: { origin: process.env.FRONTEND_URL },
path: "/socket.io/"
});
this.socket = null;
this.queue = null;
this.instance.on("connection", (socket) => {
let connectedUsersCount =
Object.keys(this.instance.sockets.sockets).length + 1;
let oneUserLeft = connectedUsersCount - 1;
console.log("New client connected ", connectedUsersCount);
// Assign socket to the class
this.socket = this.socket == null ? socket : this.socket;
/*
if (this.interval) {
clearInterval(this.interval);
}
*/
// initialize Queue
this.queue = this.queue === null ? new QueueService(socket) : this.queue;
socket.on("disconnect", () => {
console.log("Client disconnected ", oneUserLeft);
// clearInterval(this.interval);
});
});
}
QueueService
export default class QueueService {
channels: any;
socket: any;
constructor(socket: any) {
this.channels = ["integrationProgress", "news"];
this.socket = socket;
integrationQueueEvents.on("progress", (job: any) => {
console.log("Job Progressing", job);
this.socket.emit("integrationProgress", { status: true, data: job.data })
});
integrationQueueEvents.on("active", ({ jobId }) => {
console.log(`Job ${jobId} is now active`);
});
integrationQueueEvents.on("completed", ({ jobId, returnvalue }) => {
console.log(`${jobId} has completed and returned ${returnvalue}`);
this.socket.emit("integrationComplete", {
status: true,
message: returnvalue
});
});
integrationQueueEvents.on("failed", ({ jobId, failedReason }) => {
console.log(`${jobId} has failed with reason ${failedReason}`);
this.socket.emit("integrationProgress", {
status: false,
message: failedReason
});
});
}
}
Front-End
const socket = io(process.env.GATEWAY_URL, {
path: "/socket.io/"
});
socket.on("connect_error", (err) => {
console.log(`connect_error due to ${err.message}`);
socket.connect();
});
socket.on("disconnect", (socket) => {
console.log(socket);
console.log("Client disconnected ");
});
socket.on("connect", (socket) => {
console.log("Client Connected ");
console.log(socket);
});
socket.on("integrationProgress", async (socket) => {
try {
console.log(`Progress: ${socket.data}`);
updateJob(socket.data);
} catch (err) {
console.log(err);
}
});
I have a system that checks a database to see if their UserToken is in the database, If it's not it will stop the bot and display an error message, I'm trying to make the bot repeat the same function every minute to see if my database has been updated. Here is the code I'm using:
setInterval(() => {
const getToken = dtoken => new Promise((resolve, reject) => {
MongoClient.connect(url, {
useUnifiedTopology: true,
}, function(err, db) {
if (err) {
reject(err);
} else {
let dbo = db.db("heroku_fkcv4mqk");
let query = {
dtoken: dtoken
};
dbo.collection("tokens").find(query).toArray(function(err, result) {
resolve(result);
});
}
})
})
bot.on("ready", async message => {
const result = await getToken(dtoken)
if (result.length == 1) {
return
} else {
console.error('Error:', 'Your token has been revoked.')
bot.destroy()
}
})
}, 5000);
But it doesn't work and I keep getting this error message:
(node:9808) MaxListenersExceededWarning: Possible EventEmitter memory leak detected. 11 ready listeners added to [Client]. Use emitter.setMaxListeners() to increase limit
if I could get some help with the timeout that would be amazing.
Bot object listens to the event ready on each execution in setInterval(). So after every 5 seconds, a new listener is being added on bot object which you have never removed. That's why it is throwing an error that the maximum limit has been reached.
I think you can take the listener out of the setInterval. It will work.
Updated Code:::
let isReady = false;
bot.on("ready", () => {
isReady = true;
});
const getToken = dtoken => new Promise((resolve, reject) => {
MongoClient.connect(url, {
useUnifiedTopology: true,
}, function(err, db) {
if (err) {
reject(err);
} else {
let dbo = db.db("heroku_fkcv4mqk");
let query = {
dtoken: dtoken
};
dbo.collection("tokens").find(query).toArray(function(err, result) {
resolve(result);
});
}
})
})
setInterval(() => {
if (isReady) {
const result = await getToken(dtoken)
if (result.length == 1) {
return
} else {
console.error('Error:', 'Your token has been revoked.')
isReady = false
bot.destroy()
}
}
}, 5000);
I have several DBs for which i am using connection pools in node.js. Every time i refresh page i think pools are created again. i refresh page 3 times and 3 times promises resolved. i have removed several databases just to make it little bit easier to read here.
and if i un-comment connection close line my app crashes. i can't seem to figure out why
const config = require("../config/config");
const oracledb = require("oracledb");
var crm1connPromise = new Promise((resolve, reject) => {
oracledb.createPool({
user: config.crm1.user,
password: config.crm1.password,
connectString: config.crm1.connectString,
poolAlias: config.crm1.poolAlias,
poolMin: 0,
poolMax: 10,
poolTimeout: 300
}, (error, pool) => {
if (error) {
reject(err);
}
resolve("CRM1 Promise resolved")
});
});
var query2connPromise = new Promise((resolve, reject) => {
oracledb.createPool({
user: config.query2.user,
password: config.query2.password,
connectString: config.query2.connectString,
poolAlias: config.query2.poolAlias,
poolMin: 0,
poolMax: 10,
poolTimeout: 300
}, (error, pool) => {
if (error) {
reject(err);
}
resolve("QUERY2 Promise resolved --------")
});
});
var promiseArray = [crm1connPromise, crm2connPromise, crm3connPromise, crm4connPromise, csfp1connPromise, csfp2connPromise, csfp3connPromise, csfp4connPromise, cact1connPromise, cact2connPromise, cact3connPromise, cact4connPromise, cospconnPromise, cchnconnPromise, bbaseconnPromise, bcdrconnPromise, vcdbconnPromise, crptconnPromise, query2connPromise];
function getDBConnection (dbname) {
return new Promise((resolve, reject) => {
try {
Promise.all(promiseArray).then((message) => {
console.log(message);
const pool = oracledb.getPool(dbname);
pool.getConnection( (err, connection) => {
if (err) {
reject(err);
console.log(err);
}
resolve(connection);
});
});
} catch (error) {
reject(error);
}
});
}
module.exports.query = function(dbname, sql, bind = []){
return new Promise ((resolve,reject) =>{
var conn
try {
getDBConnection(dbname).then((connection) =>{
connection.execute(sql,bind,(err,result)=>{
if (err){
reject(err);
}
resolve(result);
})
//connection.close(0);
})
} catch (error) {
reject(error);
}
})
}
you can use 'Singleton'
please google 'Singleton pattern' and examples.
like this:
dataBaseManager.js:
'use strict'
var Singleton = (function () {
var instance;
function createInstance() {
var object = new dataBaseManager();
return object;
}
return {
getInstance: function () {
if (!instance) {
instance = createInstance();
}
return instance;
}
};
})();
function dataBaseManager() {
this.connected = false;
this.client = null;
this.dataBase = null;
//public methods
this.connect = function () {
try {
your_database.connect({}, (err, client) => {
if (err) {
this.connected = false;
this.client = null;
this.dataBase = null;
return;
}
this.connected = true;
this.client = client;
this.dataBase = client.db();
});
} catch (error) {
}
};
this.disconnect = function () {
try {
if (this.client) {
this.client.close();
this.connected = false;
this.client = null;
this.dataBase = null;
}
} catch (error) {
}
}
}
module.exports = Singleton;
repository.js:
const dataBaseManager = require("./dataBaseManager").getInstance();
your_get_dample_data_from_data_base_func = function (data) {
dataBaseManager.dataBase
.find({})
.toArray(function (err, result) {
if (err) {
return callback(err, null);
}
callback(null, result);
});
};
index.js:
const dataBaseManager = require("./dataBaseManager").getInstance();
function connect() {
dataBaseManager.connect();
}
function disconnect() {
dataBaseManager.disconnect();
}
Look at the node-oracledb example webappawait.js which starts the pool outside the web listener code path.
async function init() {
try {
await oracledb.createPool({
user: dbConfig.user,
password: dbConfig.password,
connectString: dbConfig.connectString
});
const server = http.createServer();
server.on('error', (err) => {
console.log('HTTP server problem: ' + err);
});
server.on('request', (request, response) => {
handleRequest(request, response);
});
await server.listen(httpPort);
console.log("Server is running at http://localhost:" + httpPort);
} catch (err) {
console.error("init() error: " + err.message);
}
}
async function handleRequest(request, response) {
. . .
}
I have a class :
class RequestTimeout {
constructor(timeoutMilliseconds) {
this.timeoutMilliseconds = timeoutMilliseconds;
this.timeoutID = undefined;
}
start() {
return new Promise((resolve, reject) => {
this.timeoutID = setTimeout(() => reject(new Error(`Request attempt exceeded timeout of ${this.timeoutMilliseconds}`)), this.timeoutMilliseconds);
});
}
clear() {
if (this.timeoutID) clearTimeout(this.timeoutID);
}
}
module.exports = RequestTimeout;
This class is used in a module:
const RequestTimeout = require('./request-timeout');
function Request() {
...
async function withTimeout(request, ms) {
const timeout = new RequestTimeout(ms);
return Promise.race([
request(),
timeout.start(),
])
.then(
response => {
timeout.clear();
return response;
},
err => {
timeout.clear();
throw err;
}
);
}
...
}
How do i mock RequestTimeout in a test using Request? For example:
it('should clear the timeout following a successful response', async () => {
nock('http://example.com')
.get('/')
.reply(200, { example: true });
const response = await request.get({ ...baseOptions });
expect(response.example).toEqual(true);
});
// MOCK
let mockGetTimeOutId = jest.fn();
jest.mock('../request-timeout', () => {
return jest.fn().mockImplementation((ms) => {
let timeoutId = undefined;
return {
start: () => new Promise((resolve, reject) => {
timeoutId = setTimeout(() => reject(), ms);
}),
clear: () => mockGetTimeOutId(timeoutId),
}
})
});
// TEST
it('should clear the timeout following a successful response', async () => {
nock('http://example.com')
.get('/')
.reply(200, { example: true });
expect(mockGetTimeOutId).toHaveBeenCalledTimes(0);
const response = await request.get({ ...baseOptions });
expect(mockGetTimeOutId).toHaveBeenCalledTimes(1);
expect(response.example).toEqual(true);
});
My code is as shown below :
rabbitmq.js
const connectRabbitMq = () => {
amqp.connect(process.env.CLOUDAMQP_MQTT_URL, function (err, conn) {
if (err) {
console.error(err);
console.log('[AMQP] reconnecting in 1s');
setTimeout(connectRabbitMq, 1);
return;
}
conn.createChannel((err, ch) => {
if (!err) {
console.log('Channel created');
channel = ch;
connection = conn;
}
});
conn.on("error", function (err) {
if (err.message !== "Connection closing") {
console.error("[AMQP] conn error", err.message);
}
});
conn.on("close", function () {
console.error("[AMQP] reconnecting");
connectRabbitMq();
});
})
};
const sendMessage = () => {
let data = {
user_id: 1,
test_id: 2
};
if (channel) {
channel.sendToQueue(q, new Buffer(JSON.stringify(data)), {
persistent: true
});
}
else {
connectRabbitMq(() => {
channel.sendToQueue(q, new Buffer(JSON.stringify(data)), {
persistent: true
});
})
}
};
const receiveMessage = () => {
if (channel) {
channel.consume(q, function (msg) {
// ch.ack(msg);
console.log(" [x] Received %s", msg.content.toString());
});
}
else {
connectRabbitMq(() => {
channel.consume(q, function (msg) {
// ch.ack(msg);
console.log(" [x] Received %s", msg.content.toString());
});
})
}
}
scheduler.js
let cron = require('node-cron');
const callMethodForeverRabbitMq = () => {
cron.schedule('*/1 * * * * *', function () {
rabbitMqClientPipeline.receiveMessage();
});
};
app.js
rabbitmq.sendMessage();
now what happens here is , the code is not able to maintain the connection alive forever . so is there any way I can keep it alive forever ?
I am not sure if you are using Promise api or callback API.
With Promise API you can do it like this:
const amqp = require('amqplib');
const delay = (ms) => new Promise((resolve => setTimeout(resolve, ms)));
const connectRabbitMq = () => amqp.connect('amqp://127.0.0.1:5672')
.then((conn) => {
conn.on('error', function (err) {
if (err.message !== 'Connection closing') {
console.error('[AMQP] conn error', err.message);
}
});
conn.on('close', function () {
console.error('[AMQP] reconnecting');
connectRabbitMq();
});
//connection = conn;
return conn.createChannel();
})
.then(ch => {
console.log('Channel created');
//channel = ch;
})
.catch((error) => {
console.error(error);
console.log('[AMQP] reconnecting in 1s');
return delay(1000).then(() => connectRabbitMq())
});
connectRabbitMq();
With callback API like this:
const amqp = require('amqplib/callback_api');
const connectRabbitMq = () => {
amqp.connect('amqp://127.0.0.1:5672', function (err, conn) {
if (err) {
console.error(err);
console.log('[AMQP] reconnecting in 1s');
setTimeout(connectRabbitMq, 1000);
return;
}
conn.createChannel((err, ch) => {
if (!err) {
console.log('Channel created');
//channel = ch;
//connection = conn;
}
});
conn.on("error", function (err) {
if (err.message !== "Connection closing") {
console.error("[AMQP] conn error", err.message);
}
});
conn.on("close", function () {
console.error("[AMQP] reconnecting");
connectRabbitMq();
});
})
};
connectRabbitMq();
UPDATE new code with request buffering
const buffer = [];
let connection = null;
let channel = null;
const connectRabbitMq = () => {
amqp.connect('amqp://127.0.0.1:5672', function (err, conn) {
if (err) {
console.error(err);
console.log('[AMQP] reconnecting in 1s');
setTimeout(connectRabbitMq, 1000);
return;
}
conn.createChannel((err, ch) => {
if (!err) {
console.log('Channel created');
channel = ch;
connection = conn;
while (buffer.length > 0) {
const request = buffer.pop();
request();
}
}
});
conn.once("error", function (err) {
channel = null;
connection = null;
if (err.message !== "Connection closing") {
console.error("[AMQP] conn error", err.message);
}
});
conn.once("close", function () {
channel = null;
connection = null;
console.error("[AMQP] reconnecting");
connectRabbitMq();
});
})
};
const sendMessage = () => {
let data = {
user_id: 1,
test_id: 2
};
if (channel) {
channel.sendToQueue(q, new Buffer(JSON.stringify(data)), {
persistent: true
});
}
else {
buffer.push(() => {
channel.sendToQueue(q, new Buffer(JSON.stringify(data)), {
persistent: true
});
});
}
};
const receiveMessage = () => {
if (channel) {
channel.consume(q, function (msg) {
// ch.ack(msg);
console.log(" [x] Received %s", msg.content.toString());
});
}
else {
buffer.push(() => {
channel.consume(q, function (msg) {
// ch.ack(msg);
console.log(" [x] Received %s", msg.content.toString());
});
})
}
};
There are edge cases where this code won't work - for example it won't reestablish queue.consume unless it's called explicitly. But overall this hopefully gives you idea on how to implement proper recovery...