I have a service in FeathersJS that initiates a connection to RabbitMQ, the issue is how to await for a channel to be ready before receiving requests:
class Service {
constructor({ amqpConnection, queueName }) {
this.amqpConnection = amqpConnection;
this.queueName = queueName;
this.replyQueueName = queueName + "Reply"
}
async create(data, params) {
new Promise(resolve => {
if (!this.channel) await this.createChannel();
channel.responseEmitter.once(correlationId, resolve);
channel.sendToQueue(this.queueName, Buffer.from(data), {
correlationId: asyncLocalStorage.getStore(),
replyTo: this.replyQueueName,
});
});
}
async createChannel() {
let connection = this.amqpConnection();
let channel = await connection.createChannel();
await channel.assertQueue(this.queueName, {
durable: false,
});
this.channel = channel;
channel.responseEmitter = new EventEmitter();
channel.responseEmitter.setMaxListeners(0);
channel.consume(
this.replyQueueName,
(msg) => {
channel.responseEmitter.emit(
msg.properties.correlationId,
msg.content.toString("utf8")
);
},
{ noAck: true }
);
}
....
}
Waiting for the channel to be created during a request seems like a waste. How should this be done "correctly"?
Feathers services can implement a setup method which will be called when the server is started (or you call app.setup() yourself):
class Service {
async setup () {
await this.createChannel();
}
}
Related
This is my first time testing RabbitMQ with node.js, and I utilized amqplib.
First, I run the
node ./messages/consumer.js
and out put as follow -:
Connected to RabbitMQ
Channel created
Waiting for messages...
Second, I run the
node ./messages/producer.js
and out put as follow -:
Connected to RabbitMQ
Channel created
Message sent: Hello, world!
Connection to RabbitMQ closed
From the RabbitMQ management console, I observed the presence of test_exchange, test_queue, and test_key, but there was no information regarding any messages. And the consumer terminal did not log any indication of receiving a message. It still displays the message "Waiting for message.". Could you kindly inform me of where I may have overlooked this information?
//config.js
module.exports = {
rabbitmq: {
host: 'localhost',
port: 5672,
username: 'guest',
password: 'guest',
vhost: '/',
exchange: 'test_exchange',
queue: 'test_queue',
routingKey: 'test_key'
}
}
//rabbitmq.js
const amqp = require("amqplib");
const config = require("../config/config");
class RabbitMQ {
constructor() {
this.connection = null;
this.channel = null;
}
async connect() {
try {
const { host, port, username, password, vhost } = config.rabbitmq;
this.connection = await amqp.connect(
`amqp://${username}:${password}#${host}:${port}/${vhost}`
);
console.log("Connected to RabbitMQ");
return this.connection;
} catch (error) {
console.error("Error connecting to RabbitMQ", error);
}
}
async createChannel() {
try {
if (!this.connection) {
await this.connect();
}
this.channel = await this.connection.createChannel();
console.log("Channel created");
return this.channel;
} catch (error) {
console.error("Error creating channel", error);
}
}
async close() {
try {
await this.connection.close();
console.log("Connection to RabbitMQ closed");
} catch (error) {
console.error("Error closing connection to RabbitMQ", error);
}
}
}
module.exports = new RabbitMQ();
//producer.js
const rabbitmq = require('../lib/rabbitmq');
const config = require('../config/config');
async function produceMessage(message) {
try {
const channel = await rabbitmq.createChannel();
const exchange = config.rabbitmq.exchange;
const queue = config.rabbitmq.queue;
const key = config.rabbitmq.routingKey;
await channel.assertExchange(exchange, 'direct', { durable: true });
await channel.assertQueue(queue, { durable: true });
await channel.bindQueue(queue, exchange, key);
const messageBuffer = Buffer.from(message);
await channel.publish(exchange, key, messageBuffer);
console.log(`Message sent: ${message}`);
await rabbitmq.close();
} catch (error) {
console.error('Error producing message', error);
}
}
produceMessage('Hello, world!');
//consumer.js
const rabbitmq = require('../lib/rabbitmq');
const config = require('../config/config');
async function consumeMessage() {
try {
const channel = await rabbitmq.createChannel();
const exchange = config.rabbitmq.exchange;
const queue = config.rabbitmq.queue;
const key = config.rabbitmq.routingKey;
await channel.assertExchange(exchange, 'direct', { durable: true });
await channel.assertQueue(queue, { durable: true });
await channel.bindQueue(queue, exchange, key);
channel.consume(queue, (msg) => {
console.log(`Message received: ${msg.content.toString()}`);
channel.ack(msg);
}, { noAck: false });
console.log('Waiting for messages...');
} catch (error) {
console.error('Error consuming message', error);
}
}
consumeMessage();
Problem:
Your message is failing to send because you are closing the connection immediately after executing the publish command. You can try this by commenting the line await rabbitmq.close(); in producer.js.
Solution:
If you want to close the connection after sending the message. You can create confirm channel instead of a normal channel, which will allow you to receive send acknowledgment.
1. Channel Creation
Change the channel creation line in rabbitmq.js file:
this.channel = await this.connection.createConfirmChannel();
2. Producer:
In the producer.js, call waitForConfirms function before closing the connection:
await channel.waitForConfirms();
await rabbitmq.close();
I am creating an event hub consumer using the example code from https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-node-get-started-send
How should I mock the events and context which are passed a arguments to the callback?
const { EventHubConsumerClient, earliestEventPosition } = require("#azure/event-hubs");
const { ContainerClient } = require("#azure/storage-blob");
const { BlobCheckpointStore } = require("#azure/eventhubs-checkpointstore-blob");
const connectionString = "EVENT HUBS NAMESPACE CONNECTION STRING";
const eventHubName = "EVENT HUB NAME";
const consumerGroup = "$Default"; // name of the default consumer group
const storageConnectionString = "AZURE STORAGE CONNECTION STRING";
const containerName = "BLOB CONTAINER NAME";
async function main() {
// Create a blob container client and a blob checkpoint store using the client.
const containerClient = new ContainerClient(storageConnectionString, containerName);
const checkpointStore = new BlobCheckpointStore(containerClient);
// Create a consumer client for the event hub by specifying the checkpoint store.
const consumerClient = new EventHubConsumerClient(consumerGroup, connectionString, eventHubName, checkpointStore);
// Subscribe to the events, and specify handlers for processing the events and errors.
const subscription = consumerClient.subscribe({
processEvents: async (events, context) => {
if (events.length === 0) {
console.log(`No events received within wait time. Waiting for next interval`);
return;
}
for (const event of events) {
console.log(`Received event: '${event.body}' from partition: '${context.partitionId}' and consumer group: '${context.consumerGroup}'`);
}
// Update the checkpoint.
await context.updateCheckpoint(events[events.length - 1]);
},
processError: async (err, context) => {
console.log(`Error : ${err}`);
}
},
{ startPosition: earliestEventPosition }
);
// After 30 seconds, stop processing.
await new Promise((resolve) => {
setTimeout(async () => {
await subscription.close();
await consumerClient.close();
resolve();
}, 30000);
});
}
main().catch((err) => {
console.log("Error occurred: ", err);
});
I assume that you want to know about the types of these two? The processEvents callback has the following signature
type ProcessEventsHandler = (
events: ReceivedEventData[],
context: PartitionContext
) => Promise<void>
Here's the ref doc for ReceivedEventData: https://learn.microsoft.com/en-us/javascript/api/#azure/event-hubs/receivedeventdata?view=azure-node-latest
and ref doc for PartitionContext: https://learn.microsoft.com/en-us/javascript/api/#azure/event-hubs/partitioncontext?view=azure-node-latest
Not too familar with jest. Here's what I tried:
async function processEvents(events, context) {
if (events.length === 0) {
console.log(`No events received within wait time. Waiting for next interval`);
return;
}
for (const event of events) {
console.log(`Received event: '${event.body}' from partition: '${context.partitionId}' and consumer group: '${context.consumerGroup}'`);
}
// Update the checkpoint.
await context.updateCheckpoint(events[events.length - 1]);
}
test('empty events', async () => {
const context = {
updateCheckpoint: jest.fn()
};
await processEvents([], context);
expect(context.updateCheckpoint.mock.calls.length).toBe(0);
});
test('non-empty events', async () => {
const context = {
updateCheckpoint: jest.fn()
};
await processEvents([{ body: "hello"}], context);
console.dir(context.updateCheckpoint.mock);
expect(context.updateCheckpoint.mock.calls.length).toBe(1);
});
I'm currently using RabbitMQ as a message broker. Recently, I see many error HeartBeat Timeout in my error log.
Also in RabbitMQ log, I see this log:
I don't know why there is too many connection from vary ranges of port. I use default setup without any further configuration.
Here is my code used to publish and consume:
import { connect } from 'amqplib/callback_api';
import hanlder from '../calculator/middleware';
import { logger } from '../config/logger';
async function consumeRabbitMQServer(serverURL, exchange, queue) {
connect('amqp://localhost', async (error0, connection) => {
if (error0) throw error0;
const channel = connection.createChannel((error1) => {
if (error1) throw error1;
});
channel.assertExchange(exchange, 'direct', {
durable: true
});
channel.assertQueue(
queue,
{
durable: true
},
(error2) => {
if (error2) throw error2;
logger.info(`Connect to ${serverURL} using queue ${queue}`);
}
);
channel.prefetch(1);
channel.bindQueue(queue, exchange, 'info');
channel.noAck = true;
channel.consume(queue, (msg) => {
hanlder(JSON.parse(msg.content.toString()))
.then(() => {
channel.ack(msg);
})
.catch((err) => {
channel.reject(msg);
});
});
});
}
export default consumeRabbitMQServer;
Code used to publish message:
import createConnection from './connection';
import { logger } from '../config/logger';
async function publishToRabbitMQServer(serverURL, exchange, queue) {
const connection = createConnection(serverURL);
const c = await connection.then(async (conn) => {
const channel = await conn.createChannel((error1) => {
if (error1) throw error1;
});
channel.assertExchange(exchange, 'direct', {
durable: true
});
channel.assertQueue(
queue,
{
durable: true
},
(error2) => {
if (error2) throw error2;
logger.info(`Publish to ${serverURL} using queue ${queue}`);
}
);
channel.bindQueue(queue, exchange, 'info');
return channel;
});
return c;
}
export default publishToRabbitMQServer;
Whenever I start my server, I run this piece of code to create a client consume to RabbitMQ:
const { RABBITMQ_SERVER } = process.env;
consumeRabbitMQServer(RABBITMQ_SERVER, 'abc', 'abc');
And this piece of code is used when ever a message in need published to RabbitMQ
const payloads = call.request.payloads;
const { RABBITMQ_SERVER } = process.env;
const channel = await publishToRabbitMQServer(RABBITMQ_SERVER, 'abc', 'abc');
for (let i = 0; i < payloads.length; i++) {
channel.publish('abc', 'info', Buffer.from(JSON.stringify(payloads[i])));
}
I'm reusing code from RabbitMQ document, and it seem that this problem happen whenever there are too many user publish message. Thanks for helping.
Update: I think the root cause is when I need to publish a message, I create a new connection. I'm working to improve it, any help is appreciate. Many thanks.
I have a scenario that on a given topic I need to consume each message one by one, do some async task and then consume the next one. I am using rabbitmq and amqp.node.
I was able to achieve this with a prefetch of 1. Which of course is not an actual solution since this would lock the whole channel and the channel have multiple topics.
So far this is my producer:
const getChannel = require("./getChannel");
async function run() {
const exchangeName = "taskPOC";
const url = "amqp://queue";
const channel = await getChannel({ url, exchangeName });
const topic = "task.init";
let { queue } = await channel.assertQueue(topic, {
durable: true
});
const max = 10;
let current = 0;
const intervalId = setInterval(() => {
current++;
if (current === max) {
clearInterval(intervalId);
return;
}
const payload = JSON.stringify({
foo: "bar",
current
});
channel.sendToQueue(queue, Buffer.from(payload), { persistent: true });
}, 3000);
}
run()
.then(() => {
console.log("Running");
})
.catch(err => {
console.log("error ", err);
});
And this is my consumer
const getChannel = require("./getChannel");
async function run() {
const exchangeName = "taskPOC";
const url = "amqp://queue";
const channel = await getChannel({ url, exchangeName });
channel.prefetch(1);
const topic = "task.init";
const { queue } = await channel.assertQueue(topic, {
durable: true
});
channel.bindQueue(queue, exchangeName, topic);
let last = new Date().getTime();
channel.consume(
queue,
msg => {
const now = new Date().getTime();
console.log(
" [x] %s %s:'%s' ",
msg.fields.routingKey,
Math.floor((now - last) / 1000),
msg.content.toString()
);
last = now;
setTimeout(function() {
channel.ack(msg);
}, 10000);
},
{ exclusive: true, noAck: false }
);
}
run()
.then(() => {
console.log("Running");
})
.catch(err => {
console.log("error ", err);
});
Is there any way on RabbitMQ to do that or I would need to handle this on my app?
Thanks.
You can use the consumer prefetch setting (see https://www.rabbitmq.com/consumer-prefetch.html). In the case of amqp node, you set this option using the prefetch function:
channel.prefetch(1, false); // global=false
In this case, each consumer on the channel will have a prefetch of 1. If you want to have different configurations for each consumer, you should create more channels.
Hope this helps.
I'm trying to work out how to close my promise based connections after publishing messages.
I've tried to extrapolate away shared code for my sender and my receiver, so I have a connection file like this:
connector.js
const amqp = require('amqplib');
class Connector {
constructor(RabbitMQUrl) {
this.rabbitMQUrl = RabbitMQUrl;
}
connect() {
return amqp.connect(this.rabbitMQUrl)
.then((connection) => {
this.connection = connection;
process.once('SIGINT', () => {
this.connection.close();
});
return this.connection.createChannel();
})
.catch( (err) => {
console.error('Errrored here');
console.error(err);
});
}
}
module.exports = new Connector(
`amqp://${process.env.AMQP_HOST}:5672`
);
Then my publisher/sender looks like this:
publisher.js
const connector = require('./connector');
class Publisher {
constructor(exchange, exchangeType) {
this.exchange = exchange;
this.exchangeType = exchangeType;
this.durabilityOptions = {
durable: true,
autoDelete: false,
};
}
publish(msg) {
connector.connect()
.then( (channel) => {
let ok = channel.assertExchange(
this.exchange,
this.exchangeType,
this.durabilityOptions
);
return ok
.then( () => {
channel.publish(this.exchange, '', Buffer.from(msg));
return channel.close();
})
.catch( (err) => {
console.error(err);
});
});
}
}
module.exports = new Publisher(
process.env.AMQP_EXCHANGE,
process.env.AMQP_TOPIC
);
But as said, I can't quite work out how to close my connection after calling publish().
You could add a close() function to connector:
close() {
if (this.connection) {
console.log('Connector: Closing connection..');
this.connection.close();
}
}
Publisher:
class Publisher {
constructor(exchange, exchangeType) {
this.exchange = exchange;
this.exchangeType = exchangeType;
this.durabilityOptions = {
durable: true,
autoDelete: false,
};
}
connect() {
return connector.connect().then( (channel) => {
console.log('Connecting..');
return channel.assertExchange(
this.exchange,
this.exchangeType,
this.durabilityOptions
).then (() => {
this.channel = channel;
return Promise.resolve();
}).catch( (err) => {
console.error(err);
});;
});
}
disconnect() {
return this.channel.close().then( () => { return connector.close();});
}
publish(msg) {
this.channel.publish(this.exchange, '', Buffer.from(msg));
};
}
Test.js
'use strict'
const connector = require('./connector');
const publisher = require('./publisher');
publisher.connect().then(() => {
publisher.publish('message');
publisher.publish('message2');
publisher.disconnect();
});