RabbitMQ have an exclusive consumer consume message serially - node.js

I have a scenario that on a given topic I need to consume each message one by one, do some async task and then consume the next one. I am using rabbitmq and amqp.node.
I was able to achieve this with a prefetch of 1. Which of course is not an actual solution since this would lock the whole channel and the channel have multiple topics.
So far this is my producer:
const getChannel = require("./getChannel");
async function run() {
const exchangeName = "taskPOC";
const url = "amqp://queue";
const channel = await getChannel({ url, exchangeName });
const topic = "task.init";
let { queue } = await channel.assertQueue(topic, {
durable: true
});
const max = 10;
let current = 0;
const intervalId = setInterval(() => {
current++;
if (current === max) {
clearInterval(intervalId);
return;
}
const payload = JSON.stringify({
foo: "bar",
current
});
channel.sendToQueue(queue, Buffer.from(payload), { persistent: true });
}, 3000);
}
run()
.then(() => {
console.log("Running");
})
.catch(err => {
console.log("error ", err);
});
And this is my consumer
const getChannel = require("./getChannel");
async function run() {
const exchangeName = "taskPOC";
const url = "amqp://queue";
const channel = await getChannel({ url, exchangeName });
channel.prefetch(1);
const topic = "task.init";
const { queue } = await channel.assertQueue(topic, {
durable: true
});
channel.bindQueue(queue, exchangeName, topic);
let last = new Date().getTime();
channel.consume(
queue,
msg => {
const now = new Date().getTime();
console.log(
" [x] %s %s:'%s' ",
msg.fields.routingKey,
Math.floor((now - last) / 1000),
msg.content.toString()
);
last = now;
setTimeout(function() {
channel.ack(msg);
}, 10000);
},
{ exclusive: true, noAck: false }
);
}
run()
.then(() => {
console.log("Running");
})
.catch(err => {
console.log("error ", err);
});
Is there any way on RabbitMQ to do that or I would need to handle this on my app?
Thanks.

You can use the consumer prefetch setting (see https://www.rabbitmq.com/consumer-prefetch.html). In the case of amqp node, you set this option using the prefetch function:
channel.prefetch(1, false); // global=false
In this case, each consumer on the channel will have a prefetch of 1. If you want to have different configurations for each consumer, you should create more channels.
Hope this helps.

Related

How to do a ioredis scanStream on node-redis in TypeScript?

The Following code works on ioredis scanstream. I am trying to do it on node-redis in typescript.
It scans a stream and returns gameIds to an Express client, when it reaches 'end' it returns the results to the client.
const stream = redis.scanStream({
match: "kaboom:moves:*",
});
const gameIds: any = [];
This is my attempt:
const stream = client.scanIterator({
TYPE: "string", // `SCAN` only
MATCH: "kaboom:moves:*",
COUNT: 100,
});
const LIMIT = 3;
const asyncIterable = {
[Symbol.asyncIterator]() {
let i = 0;
return {
next() {
const done = i === LIMIT;
const value = done ? undefined : i++;
return Promise.resolve({ value, done });
},
return() {
// This will be reached if the consumer called 'break' or 'return' early in the loop.
return { done: true };
},
};
},
};
(async () => {
for await (const keys of stream) {
gameIds.push(keys.split(":")[2]);
}
})();
const gameIds: any = [];
How do I write this part so that it will work with node-redis in typescript?
stream.on('data', (keys: any) => {
//Extract the gameId from the key and append to gameIds array.
keys.forEach((key: any) => gameIds.push(key.split(':')[2]));
});
stream.on('end', () => {
res.status(200).json({
data: {
gameIds,
length: gameIds.length
},
status: 'success',
})
})
ScanIterator was correct.
const stream = client.scanIterator({
TYPE: 'string', // `SCAN` only
MATCH: 'kaboom:moves:*',
COUNT: 100
});
const LIMIT = 3;
const asyncIterable = {
[Symbol.asyncIterator]() {
let i = 0;
return {
next() {
const done = i === LIMIT;
const value = done ? undefined : i++;
return Promise.resolve({ value, done });
},
return() {
// This will be reached if the consumer called 'break' or 'return' early in the loop.
return { done: true };
}
};
}
};
(async () => {
for await (const keys of stream) {
gameIds.push(keys.split(':')[2]);
}
})();

clearInterval of a external function not working - Node.JS

I have a setInterval function that's been called in another function, and I need to stop it when the proccess is done. I tried to set this setInterval function as a variable and call clearInterval, but the interval keeps running
const createInterval = (visibilityTimeout, startDateTime, message) => {
setInterval(() => {
const currentDateTime = moment().valueOf();
const timeDifference = (visibilityTimeout * 1000) - (currentDateTime - startDateTime);
if (timeDifference >= 600000) {
return;
}
if (timeDifference < 494983) {
const params = {
QueueUrl: 'http://localhost:4566/000000000000/test-queue2',
ReceiptHandle: message.ReceiptHandle,
VisibilityTimeout: visibilityTimeout,
};
sqs.changeMessageVisibility(params, (err, data) => {
if (err) logger.error(err, err.stack);
else logger.info(data);
});
// eslint-disable-next-line no-param-reassign
visibilityTimeout += 300;
}
}, 5000);
};
module.exports = async (message) => {
const startDateTime = moment().valueOf();
const {
noteId,
} = JSON.parse(message.Body);
logger.info(`Processing message [noteId=${noteId}]`);
try {
const note = await TestSessionNote.findById(noteId);
const testSession = await TestSession.findById(note.test_session_id);
logger.info(`Downloading video [key=${testSession.video_key}]`);
const isProcessing = true;
const interval = createInterval(500, startDateTime, message, isProcessing);
await sleep(20000);
clearInterval(interval);
logger.info(`Finished processing message [noteId=${noteId}]`);
} catch (ex) {
await TestSessionNote.update(noteId, { status: 'transcribe_error' });
logger.error(`Error processing message [noteId=${noteId}]`, ex);
}
};
I know that if i create a var test = setInterval(() => {console.log('blabla')}, 500) and call clearInterval(test) it works, but i don't know how can i do this calling a function
I think that you have to return from createInterval function the intervalId and after that it should work.
Can you check what value has your intervalId right now, with your current implementation?
https://developer.mozilla.org/en-US/docs/Web/API/setInterval
"The returned intervalID is a numeric, non-zero value which identifies the timer created by the call to setInterval(); this value can be passed to clearInterval() to cancel the interval."

RabbitMQ HeartBeat Timeout issue

I'm currently using RabbitMQ as a message broker. Recently, I see many error HeartBeat Timeout in my error log.
Also in RabbitMQ log, I see this log:
I don't know why there is too many connection from vary ranges of port. I use default setup without any further configuration.
Here is my code used to publish and consume:
import { connect } from 'amqplib/callback_api';
import hanlder from '../calculator/middleware';
import { logger } from '../config/logger';
async function consumeRabbitMQServer(serverURL, exchange, queue) {
connect('amqp://localhost', async (error0, connection) => {
if (error0) throw error0;
const channel = connection.createChannel((error1) => {
if (error1) throw error1;
});
channel.assertExchange(exchange, 'direct', {
durable: true
});
channel.assertQueue(
queue,
{
durable: true
},
(error2) => {
if (error2) throw error2;
logger.info(`Connect to ${serverURL} using queue ${queue}`);
}
);
channel.prefetch(1);
channel.bindQueue(queue, exchange, 'info');
channel.noAck = true;
channel.consume(queue, (msg) => {
hanlder(JSON.parse(msg.content.toString()))
.then(() => {
channel.ack(msg);
})
.catch((err) => {
channel.reject(msg);
});
});
});
}
export default consumeRabbitMQServer;
Code used to publish message:
import createConnection from './connection';
import { logger } from '../config/logger';
async function publishToRabbitMQServer(serverURL, exchange, queue) {
const connection = createConnection(serverURL);
const c = await connection.then(async (conn) => {
const channel = await conn.createChannel((error1) => {
if (error1) throw error1;
});
channel.assertExchange(exchange, 'direct', {
durable: true
});
channel.assertQueue(
queue,
{
durable: true
},
(error2) => {
if (error2) throw error2;
logger.info(`Publish to ${serverURL} using queue ${queue}`);
}
);
channel.bindQueue(queue, exchange, 'info');
return channel;
});
return c;
}
export default publishToRabbitMQServer;
Whenever I start my server, I run this piece of code to create a client consume to RabbitMQ:
const { RABBITMQ_SERVER } = process.env;
consumeRabbitMQServer(RABBITMQ_SERVER, 'abc', 'abc');
And this piece of code is used when ever a message in need published to RabbitMQ
const payloads = call.request.payloads;
const { RABBITMQ_SERVER } = process.env;
const channel = await publishToRabbitMQServer(RABBITMQ_SERVER, 'abc', 'abc');
for (let i = 0; i < payloads.length; i++) {
channel.publish('abc', 'info', Buffer.from(JSON.stringify(payloads[i])));
}
I'm reusing code from RabbitMQ document, and it seem that this problem happen whenever there are too many user publish message. Thanks for helping.
Update: I think the root cause is when I need to publish a message, I create a new connection. I'm working to improve it, any help is appreciate. Many thanks.

AMQP + NodeJS wait for channel

I have a service in FeathersJS that initiates a connection to RabbitMQ, the issue is how to await for a channel to be ready before receiving requests:
class Service {
constructor({ amqpConnection, queueName }) {
this.amqpConnection = amqpConnection;
this.queueName = queueName;
this.replyQueueName = queueName + "Reply"
}
async create(data, params) {
new Promise(resolve => {
if (!this.channel) await this.createChannel();
channel.responseEmitter.once(correlationId, resolve);
channel.sendToQueue(this.queueName, Buffer.from(data), {
correlationId: asyncLocalStorage.getStore(),
replyTo: this.replyQueueName,
});
});
}
async createChannel() {
let connection = this.amqpConnection();
let channel = await connection.createChannel();
await channel.assertQueue(this.queueName, {
durable: false,
});
this.channel = channel;
channel.responseEmitter = new EventEmitter();
channel.responseEmitter.setMaxListeners(0);
channel.consume(
this.replyQueueName,
(msg) => {
channel.responseEmitter.emit(
msg.properties.correlationId,
msg.content.toString("utf8")
);
},
{ noAck: true }
);
}
....
}
Waiting for the channel to be created during a request seems like a waste. How should this be done "correctly"?
Feathers services can implement a setup method which will be called when the server is started (or you call app.setup() yourself):
class Service {
async setup () {
await this.createChannel();
}
}

Kafka consumer not consuming message after pause and resume

I am using this node-rdkafka library to implement the node kafka with consumer pause and resume method to handle the back-pressure. I have created the small demo where I can pause the consumer and resume the consumer but the problem is after resume the consumer It stops the consuming messages.
Here is my code.
const Kafka = require('node-rdkafka');
const topic = 'create_user_channel';
const log_divider = '-----------------------------------';
const consumer = new Kafka.KafkaConsumer({
'group.id':'gsuite_consumer',
'metadata.broker.list': '*******',
'sasl.mechanisms': 'PLAIN',
'sasl.username': '********',
'sasl.password': '********',
'security.protocol': 'SASL_SSL',
'enable.auto.commit':false
}, {});
// Connect the consumer.
consumer.connect({timeout: "1000ms"}, (err) => {
if (err) {
console.log(`Error connecting to Kafka broker: ${err}`);
process.exit(-1);
}
console.log("Connected to Kafka broker");
});
consumer.on('disconnected', (args) => {
console.error(`Consumer got disconnected: ${JSON.stringify(args)}`);
});
let max_queue_size = 3;
let current_queue = [];
let is_pause = false;
// register ready handler.
consumer.on('ready', (arg)=>{
console.log('consumer ready.' + JSON.stringify(arg));
console.log('Consumer is ready');
consumer.subscribe([topic]);
setInterval(function() {
console.log('consumer has consume on :'+timeMs());
consumer.consume();
}, 1000);
});
consumer.on('data',async (data)=>{
console.log('************consumer is consuming data***********:'+timeMs());
if(!is_pause) {
is_pause = true;
if(data && typeof data !== 'undefined') {
try {
console.log('consumer has received the data:'+timeMs());
consumer.pause([topic]);
console.log('consumer has pause the consuming:'+timeMs());
await processMessage(data);
console.log('consumer is resumed:'+timeMs());
consumer.resume([topic]);
console.log(log_divider);
is_pause = false;
} catch(error) {
console.log('data consuming error');
console.log(error);
}
} else {
is_pause = false;
}
}
});
async function processMessage(data) {
// await print_bulk(data);
await processData(0,data);
}
async function print_bulk(data) {
for(var i=0;i<data.length;i++) {
await processData(i,data[i]);
}
}
/**
* Wait specified number of milliseconds.
* #param ms
*/
async function wait(ms) {
console.log('wait for the 3 sec');
return new Promise((resolve) => setTimeout(resolve, ms));
}
var timeMs = ()=> {
var d = new Date();
var h = addZero(d.getHours(), 2);
var m = addZero(d.getMinutes(), 2);
var s = addZero(d.getSeconds(), 2);
var ms = addZero(d.getMilliseconds(), 3);
return h + ":" + m + ":" + s + ":" + ms;
}
var addZero = (x, n)=> {
while (x.toString().length < n) {
x = "0" + x;
}
return x;
}
async function processData(i,m) {
if (m) {
console.log('processing a data start:'+timeMs());
console.log('Received a message:');
console.log(' message: ' + m.value.toString());
console.log(' key: ' + m.key);
console.log(' size: ' + m.size);
console.log(' topic: ' + m.topic);
console.log(' offset: ' + m.offset);
console.log(' partition: ' + m.partition);
consumer.commitMessage(m);
}
await wait(3000);
console.log('process a data completed:'+timeMs());
// delete current_queue[i];
// console.log('after delting lenght of current queue:'+current_queue.length);
// console.log(log_divider);
return true;
}
can anybody help me, what I am doing wrong while resuming the consumer? When I start the consumer it just receives only one message and after resuming it still not consuming any further messages.
I have figure out the issue. Along with consumer.pause() & consumer.resume() method I need to use the consumer.assignments() method as well.
so It will be like this
consumer.pause(consumer.assignments());
consumer.resume(consumer.assignments());

Resources