The below code snippet is working fine. But sometimes it's not reading messages from Kafka's topic. I am not getting any errors. At Kafka's side(we are using Aiven Managed Kafka), the consumer group has associated with the topic and consumer script is running nicely.
I need your guidance to resolve the above issue.
Kafka Version - 2.0.1
Node Module Version - "node-rdkafka": "^2.7.0"
Code Ref - https://github.com/Blizzard/node-rdkafka
const Kafka = require('node-rdkafka');
const path = require('path');
console.log(Kafka.features);
const consumer = new Kafka.KafkaConsumer({
'group.id': 'testxyz',
'metadata.broker.list': "kafka-production-url",
'security.protocol': 'SSL',
'ssl.key.password': "password",
'ssl.key.location': '/var/www/html/config/service.key',
'ssl.certificate.location': '/var/www/html/config/service.cert',
'ssl.ca.location': '/var/www/html/config/ca.pem',
'socket.keepalive.enable': true,
'enable.auto.commit': false,
'rebalance_cb': function(err, assignment) {
if (err.code === Kafka.CODES.ERRORS.ERR__ASSIGN_PARTITIONS) {
// Note: this can throw when you are disconnected. Take care and wrap it in
// a try catch if that matters to you
this.assign(assignment);
} else if (err.code == Kafka.CODES.ERRORS.ERR__REVOKE_PARTITIONS){
// Same as above
this.unassign();
} else {
// We had a real error
console.error(err);
}
},
'offset_commit_cb': function (err: any, topicPartitions: any) {
if (err) {
// There was an error committing
console.error("error in commit", err);
} else {
// Commit went through. Let's log the topic partitions
console.log("Success in commit", topicPartitions);
}
}
},
'auto.offset.reset': 'earliest');
consumer.on('ready', {
consumer.subscribe("mytopic");
consumer.consume();
});
// handle the messgae
consumer.on('data', {
console.log(data);
});
consumer.on('disconnected', {
console.log("onDisconnected",args);
});
consumer.connect();
Related
Im trying to use proto messages with protobuf.js, encode them and send them to a RabbitMQ message broker. I have the following sub-folders inside my project:
- model
- protos
- transactions.proto
- RabitMQ.js
- routes
- rmq-api.js
I added a route which does the following(Using express) in the rmq-api.js file:
const RabbitMQ = require('../model/RabbitMQ');
router.post('/api/transactions' ,function (req,res,next) {
RabbitMQ.PublishTransactionsMessage(DummyMessage).then(() => {
res.status(200).send({message: "OK :)"});
}).catch((e) => {
res.status(500).send({error:e.message});
});
});
In the RabitMQ.js file I have the following code:
module.exports = {
PublishTransactionsMessage: function(message) {
return new Promise((resolve, reject) => {
amqp.connect(RabbitMQConfig.url, function (error, connection) {
if (error) {
console.error("Could not connect to the rabbit message broker on {0} - " +
"Check connection please and try again".format(RabbitMQConfig.url));
console.error("Error message - {0}".format(error));
reject(error)
}
connection.createChannel(function(error, channel) {
if (error) {
console.error("Could Create channel - {0}".format(error.message));
reject(error)
}
const queue = RabbitMQConfig.queue;
channel.assertQueue(queue, {
durable: true
});
// Convert Message to protobuff
protobuf.load("./protos/transactions.proto").then((err, root) => {
if (err) {
reject(err);
}
let ScraperMessageResult = root.lookupType("transactions.ScraperMessageResult");
const errMsg = ScraperMessageResult.verify(message);
if (errMsg)
reject(errMsg);
let buffer = ScraperMessageResult.encode(message).finish();
channel.sendToQueue(queue, buffer);
console.log(`Sent ${message} to queue: ${queue}`);
resolve()
}).catch((err) => {
reject(err);
});
});
});
});
},
};
In the code shown above in the line:
protobuf.load("./protos/transactions.proto").then((err, root) => {
I keep catching the following error:
Inside this catch block:
}).catch((err) => {
reject(err);
});
This seems like a pretty simple problem however I havent found anything on this online so I might be missing something really simple here.
P.S. I tried using __dirname + "/protos/transaction.proto" and still couldnt get this to work.
Please help me figure this out.
The problem is that your function doesn't look at the dist directory,
await protocolBuffer.load(__dirname + '/item.proto'); should look there,
and also you need to copy the file manuly by using copy/cp command, you can add it as part of the package json script like so:
"build": "nest build && COPY src\\reading_list\\protocol_buffer\\*.proto dist\\reading_list\\protocol_buffer\\"
I'm using RTCMulticonnection MultiRTC script to capture and stream multiple user cameras.
I guess, If any user refresh page then session keeps alive in background even I've added page unload event
window.onbeforeunload = function() {
rtcMultiConnection.close();
};
my problem is that joining a room after refresh keeps throwing error/warning message Session-Descriptions not found. Rechecking...
Why session description not found? I've checked RTCMulticonnection js and this error is throwing from below function.
function joinSession(session, joinAs) {
if (isString(session)) {
connection.skipOnNewSession = true;
}
console.log(session);
console.log(joinAs);
if (!rtcMultiSession) {
log('Signaling channel is not ready. Connecting...');
// connect with signaling channel
initRTCMultiSession(function() {
log('Signaling channel is connected. Joining the session again...');
setTimeout(function() {
joinSession(session, joinAs);
}, 1000);
});
return;
}
// connection.join('sessionid');
if (isString(session)) {
if (connection.sessionDescriptions[session]) {
session = connection.sessionDescriptions[session];
} else
return setTimeout(function() {
log('Session-Descriptions not found. Rechecking..');
joinSession(session, joinAs);
}, 1000);
}
// connection.join('sessionid', { audio: true });
if (joinAs) {
return captureUserMedia(function() {
session.oneway = true;
joinSession(session);
}, joinAs);
}
if (!session || !session.userid || !session.sessionid) {
error('missing arguments', arguments);
var error = 'Invalid data passed over "connection.join" method.';
connection.onstatechange({
userid: 'browser',
extra: {},
name: 'Unexpected data detected.',
reason: error
});
throw error;
}
if (!connection.dontOverrideSession) {
connection.session = session.session;
}
var extra = connection.extra || session.extra || {};
// todo: need to verify that if-block statement works as expected.
// expectations: if it is oneway streaming; or if it is data-only connection
// then, it shouldn't capture user-media on participant's side.
if (session.oneway || isData(session)) {
rtcMultiSession.joinSession(session, extra);
} else {
captureUserMedia(function() {
rtcMultiSession.joinSession(session, extra);
});
}
}
Upgraded my application with RTCMulticonnection version v3, also used socket.io instead of WebSocket, earlier I was using WebSocket.
I am working on a node app that essentially is a simple AWS SQS poller that should sit and listen to new items in different queues.
Here is my module.export:
module.exports = {
readMessage: function(qParams, qType, tableName) {
logger.debug(qType);
SQS.receiveMessage(qParams, handleSqsResponse);
function handleSqsResponse (err, data) {
if(err) logger.error("handleSqsResponse error:" + err);
if (data && data.Messages) {
data.Messages.forEach(processMessage)
readMessage(); // continue reading until draining the queue (or UPTIME reached)
}
else{
logger.debug("no data in sqs.");
// process.exit();
}
}
// 'processing' is mainly writing to logs using winston. Could add here any transformations and transmission to remote systems
function processMessage(sqsMessage){
// Parse sqs messag
var msgObj = JSON.parse(sqsMessage.Body);
// Process
logger.info(msgObj.Message);
_.extend(qParams, { "ReceiptHandle": sqsMessage.ReceiptHandle });
dbMap[qType](msgObj, qParams, tableName);
}
}
}
The issue I am running into is when I attempt to call readMessage(); again. I get the error of ReferenceError: readMessage is not defined
module.exports is a plain object that is exposed to outer modules that has a method readMessage. readMessage() should be module.exports.readMessage().
Also i would suggest creating a variable and then exporting that:
var obj = {
readMessage: function(qParams, qType, tableName) {
logger.debug(qType);
SQS.receiveMessage(qParams, handleSqsResponse);
function handleSqsResponse (err, data) {
if(err) logger.error("handleSqsResponse error:" + err);
if (data && data.Messages) {
data.Messages.forEach(processMessage)
obj.readMessage(); // continue reading until draining the queue (or UPTIME reached)
}
else{
logger.debug("no data in sqs.");
// process.exit();
}
}
// 'processing' is mainly writing to logs using winston. Could add here any transformations and transmission to remote systems
function processMessage(sqsMessage){
// Parse sqs messag
var msgObj = JSON.parse(sqsMessage.Body);
// Process
logger.info(msgObj.Message);
_.extend(qParams, { "ReceiptHandle": sqsMessage.ReceiptHandle });
dbMap[qType](msgObj, qParams, tableName);
}
}
}
module.exports = obj;
Please note that I only responded to the question you specifically asked. I didn't take into account any architectural issue associate with the code.
function functionName(has = false){
var total = 0;
if(has){
functionName(true)
} else {
// Todo
}
}
module.exports.functionName = functionName;
I want to receive a message after a certain amount of time in one of my workers. I decided to go with Node and RabbitMQ after discovering so-called dead letter exchanges.
The message seems to get send to the queue in DeadExchange, but the consumer is never receiving the message after the elapsed time in the WorkQueue in the WorkExchange. Either the bindQueue is off, or the dead-letter'ing doesn't work?
I've tried a lot of different values now. Can someone please point out what I'm missing?
var amqp = require('amqplib');
var url = 'amqp://dev.rabbitmq.com';
amqp.connect(url).then(function(conn) {
//Subscribe to the WorkQueue in WorkExchange to which the "delayed" messages get dead-letter'ed (is that a verb?) to.
return conn.createChannel().then(function(ch) {
return ch.assertExchange('WorkExchange', 'direct').then(function() {
return ch.assertQueue('WorkQueue', {
autoDelete: false,
durable: true
})
}).then(function() {
return ch.bindQueue('WorkQueue', 'WorkExchange', '');
}).then(function() {
console.log('Waiting for consume.');
return ch.consume('WorkQueue', function(msg) {
console.log('Received message.');
console.log(msg.content.toString());
ch.ack(msg);
});
});
})
}).then(function() {
//Now send a test message to DeadExchange to a random (unique) queue.
return amqp.connect(url).then(function(conn) {
return conn.createChannel();
}).then(function(ch) {
return ch.assertExchange('DeadExchange', 'direct').then(function() {
return ch.assertQueue('', {
arguments: {
'x-dead-letter-exchange': 'WorkExchange',
'x-message-ttl': 2000,
'x-expires': 10000
}
})
}).then(function(ok) {
console.log('Sending delayed message');
return ch.sendToQueue(ok.queue, new Buffer(':)'));
});
})
}).then(null, function(error) {
console.log('error\'ed')
console.log(error);
console.log(error.stack);
});
I'm using amqp.node (https://github.com/squaremo/amqp.node) which is amqplib in npm. Although node-amqp (https://github.com/postwait/node-amqp) seems to be so much more popular, it doesn't implement the full protocol and there are quite some outstanding issues regarding reconnecting.
dev.rabbitmq.com is running RabbitMQ 3.1.3.
This is a working code.When a message spends more than ttl in DeadExchange, it is pushed to WorkExchange. The key to success is defining the right routing key. The exchange-queue to which you wish to send post ttl, should be bounded with a routing key(note: not default), and 'x-dead-letter-routing-key' attributes value should match that route-key.
var amqp = require('amqplib');
var url = 'amqp://localhost';
amqp.connect(url).then(function(conn) {
//Subscribe to the WorkQueue in WorkExchange to which the "delayed" messages get dead-letter'ed (is that a verb?) to.
return conn.createChannel().then(function(ch) {
return ch.assertExchange('WorkExchange', 'direct').then(function() {
return ch.assertQueue('WorkQueue', {
autoDelete: false,
durable: true
})
}).then(function() {
return ch.bindQueue('WorkQueue', 'WorkExchange', 'rk1');
}).then(function() {
console.log('Waiting for consume.');
return ch.consume('WorkQueue', function(msg) {
console.log('Received message.');
console.log(msg.content.toString());
ch.ack(msg);
});
});
})
}).then(function() {
//Now send a test message to DeadExchange to DEQ queue.
return amqp.connect(url).then(function(conn) {
return conn.createChannel();
}).then(function(ch) {
return ch.assertExchange('DeadExchange', 'direct').then(function() {
return ch.assertQueue('DEQ', {
arguments: {
'x-dead-letter-exchange': 'WorkExchange',
'x-dead-letter-routing-key': 'rk1',
'x-message-ttl': 15000,
'x-expires': 100000
}
})
}).then(function() {
return ch.bindQueue('DEQ', 'DeadExchange', '');
}).then(function() {
console.log('Sending delayed message');
return ch.publish('DeadExchange', '', new Buffer("Over the Hills and Far Away!"));
});
})
}).then(null, function(error) {
console.log('error\'ed')
console.log(error);
console.log(error.stack);
});
Here's an example using AMQP Connection Manager for Node. I noticed no examples seemed to match what we were doing in our code, so I made a repo with a simple example and one with retry counts via republishing back to the main exchange: https://github.com/PritchardAlexander/node-amqp-dead-letter-queue
Here's the simple example:
const amqp = require('amqp-connection-manager');
const username = encodeURIComponent('queue');
const password = encodeURIComponent('pass');
const port = '5672';
const host = 'localhost';
const connectionString = `amqp://${username}:${password}#${host}:${port}`;
// Ask the connection manager for a ChannelWrapper. Specify a setup function to
// run every time we reconnect to the broker.
connection = amqp.connect([connectionString]);
// A channel is your ongoing connection to RabbitMQ.
// All commands go through your channel.
connection.createChannel({
json: true,
setup: function (channel) {
channel.prefetch(100);
// Setup EXCHANGES - which are hubs you PUBLISH to that dispatch MESSAGES to QUEUES
return Promise.all([
channel.assertExchange('Test_MainExchange', 'topic', {
durable: false,
autoDelete: true,
noAck: false
}),
channel.assertExchange('Test_DeadLetterExchange', 'topic', {
durable: false,
autoDelete: true,
maxLength: 1000,
noAck: true // This means dead letter messages will not need an explicit acknowledgement or rejection
})
])
// Setup QUEUES - which are delegated MESSAGES by EXCHANGES.
// The MESSAGES then need to be CONSUMED.
.then(() => {
return Promise.all([
channel.assertQueue(
'Test_MainQueue',
options = {
durable: true,
autoDelete: true,
exclusive: false,
messageTtl: 1000*60*60*1,
deadLetterExchange: 'Test_DeadLetterExchange'
}
),
channel.assertQueue('Test_DeadLetterQueue',
options = {
durable: false,
autoDelete: true,
exclusive: false
}
)
]);
})
// This glues the QUEUES and EXCHANGES together
// The last parameter is a routing key. A hash/pound just means: give me all messages in the exchange.
.then(() => {
return Promise.all([
channel.bindQueue('Test_MainQueue', 'Test_MainExchange', '#'),
channel.bindQueue('Test_DeadLetterQueue', 'Test_DeadLetterExchange', '#')
]);
})
// Setup our CONSUMERS
// They pick MESSAGES off of QUEUES and do something with them (either ack or nack them)
.then(() => {
return Promise.all([
channel.consume('Test_MainQueue', (msg) => {
const stringifiedContent = msg.content ? msg.content.toString() : '{}';
console.log('Test_MainQueue::CONSUME ' + stringifiedContent);
const messageData = JSON.parse(stringifiedContent);
if (messageData.value === 0) {
console.log('Test_MainQueue::REJECT ' + stringifiedContent);
// the 'false' param at the very end means, don't retry! dead letter this instead!
return channel.nack(msg, true, false);
}
return channel.ack(msg);
})
]),
channel.consume('Test_DeadLetterQueue', (msg) => {
const stringifiedContent = msg.content ? msg.content.toString() : '{}';
console.log('');
console.log('Test_DeadLetterQueue::CONSUME ' + stringifiedContent);
console.log('');
});
})
.then(() => {
setInterval(function () {
const messageData = {
text: 'Dead letter if 0',
value: Math.floor(Math.random()*5)
};
const stringifiedMessage = JSON.stringify(messageData);
// Publish message to exchange
if (channel.publish('Test_MainExchange', '', new Buffer(stringifiedMessage))) {
console.log(`Sent ${stringifiedMessage}`);
} else {
console.log(`Failed to send ${stringifiedMessage}`);
};
}, 300);
});
}
});
There was a bug in Channel#assertQueue in AMQP.Node which just got fixed, see https://github.com/squaremo/amqp.node/commit/3749c66b448875d2df374e6a89946c0bdd0cb918. The fix is on GitHub but not in npm just yet.
I am using the following code for connecting the CouchBase
couchbase.connect(config.CouchBaseConnector, function (err, CouchBaseDB) {
if (err) {
throw (err)
}
CouchBaseDB.set(keyPush, docPush, function (err, meta) {
if (err) { console.log(err); }
});
}
But its creating multiple number of connection.
Can someone help me out to fix the issue. Basically I want to do something like connection pool and keep re-using.
I came across a doc from CouchBase regarding the same. But not able to figure it out how exactly it works and the steps to deploy the same on windows 7 64-bit version.
Update:
I think moxi-server is not released for Windows OS as of now.
The Couchbase Node SDK is a Connection Pool itself. It is responsible of managing connection to the cluster, be alerted about any change in the server topology (add/remove/failed nodes)
This is why most of the time you put your code in a global callback method and reuse the connection
var express = require('express'),
driver = require('couchbase'),
routes = require('./routes');
dbConfiguration = {
"hosts": ["my-couchbase-server:8091"],
"bucket": "bucket"
}
driver.connect(dbConfiguration, function(err, cb) {
if (err) {
throw (err)
}
// put your application logic here
});
If you want to use a global variable you need to wait for the callback and be sure that the connection is established before using it.
I found the following code is working for me.
Please anyone has better solution please post it, I always welcome that.
GLOBAL.CouchBaseDBConnection = undefined;
function OpenCouchBase(callback) {
if (CouchBaseDBConnection == undefined) {
couchbase.connect(config.CouchBaseConnector, function (err, couchbaseOpenCon) {
if (err)
return console.log("Failed to connect to the CouchBase");
else {
CouchBaseDBConnection = couchbaseOpenCon
callback(null, couchbaseOpenCon);
}
});
}
else { callback(null, CouchBaseDBConnection); }
};
module.exports.OpenPoolCouchBase = OpenCouchBase;
You can use generic resource pool module for Node: node-pool
It is generic pool, so you can adapt it for your needs.
EDIT:
Here is example code:
var poolModule = require('generic-pool');
var pool = poolModule.Pool({
name : 'couch',
create : function(callback) {
couchbase.connect(config.CouchBaseConnector, function (err, couchbaseOpenCon) {
if (err)
return console.log("Failed to connect to the CouchBase");
else {
CouchBaseDBConnection = couchbaseOpenCon
callback(null, couchbaseOpenCon);
}
});
},
destroy : function(client) { client.end(); },
max : 10,
// specifies how long a resource can stay idle in pool before being removed
idleTimeoutMillis : 30000,
// if true, logs via console.log - can also be a function
log : true
});
// acquire connection - callback function is called once a resource becomes available
pool.acquire(function(err, client) {
if (err) {
// handle error - this is generally the err from your
// factory.create function
}
else {
console.log("do whatever you want with the client ...");
pool.release(client);
});
}
});