Node kafka consumer to process messages in sequence - node.js

Question

I have a kafka topic that I want to consume with a node app. The node app must process the messages from the topic in sequence, one by one, not many at the same time.
I tried this kind of code but this is not doing what I want. When there is messages in the topic waiting for processing and this code is started the on 'message' event gets triggered immediately for all the the messages. The first message gets mutex lock first but the rest of the messages are processed in random order.
var mutex = require( 'node-mutex' )();
var crypto = require('crypto');
var mutexToken = crypto.randomBytes(64).toString('hex');
var kafka = require('kafka-node');
var Consumer = kafka.Consumer;
var client = new kafka.Client('localhost:2181');
var consumer = new Consumer(
client,
[
{ topic: 'my_topic' }
]
);
consumer.on('message', function(message) {
console.log("new message")
mutex
.lock( mutexToken )
.then( function( unlock ) {
console.log(message);
unlock();
} );
});
Is it possible to consume the messages one by one, synchronously? Maybe with some other library?

Answer 1

I believe you can control the message offset directly by explicitly disabling the autoCommit feature.
Here is the link to the consumer documentation:
https://www.npmjs.com/package/kafka-node#highlevelconsumer
Here is a link to an example with autoCommit set off:
https://github.com/SOHU-Co/kafka-node/blob/master/example/consumer.js
https://github.com/SOHU-Co/kafka-node/blob/master/example/offset.js
I have never needed to disable the autoCommit feature, so I can't speak to the implementation.
From the test code it looks like it should be something along these lines:
var Offset = kafka.Offset;
var offset = new Offset(client);
var topics = [ { topic: EXISTS_TOPIC_2 } ];
var options = { autoCommit: false, groupId: '_groupId_1_test' };
var consumer = new Consumer(client, topics, options);
var count = 0;
consumer.on('error', noop);
consumer.on('offsetOutOfRange', function (topic) {
offsetOutOfRange(topic, this);
});
consumer.on('message', function (message) {
message.topic.should.equal(EXISTS_TOPIC_2);
message.value.should.equal('hello kafka');
message.partition.should.equal(0);
offset.commit('_groupId_1_test', [message], function (err) {
if (count++ === 0) done(err);
});
});
OK I looked at the API a little more and thought this might eb an angle worth investigating for you:
Consumer.prototype.pauseTopics = function (topics) {
if (!this.pausedPayloads) this.pausedPayloads = [];
pauseOrResume(this.payloads, this.pausedPayloads, topics);
};
Consumer.prototype.resumeTopics = function (topics) {
if (!this.pausedPayloads) this.pausedPayloads = [];
var reFetch = !this.payloads.length;
pauseOrResume(this.pausedPayloads, this.payloads, topics);
reFetch = reFetch && this.payloads.length;
if (reFetch) this.fetch();
};
from the documentation:
pause()
Pause the consumer. Calling pause does not automatically stop messages
from being emitted. This is because pause just stops the kafka
consumer fetch loop. Each iteration of the fetch loop can obtain a
batch of messages (limited by fetchMaxBytes).
So if you only fetch one message (perhaps your bytes are small enough that a max fetch is only 1), then pause will stop the next fetch from happening. But if you fetched multiple messages, pause will not prevent more than one message being emitted.
I think to be 100% certain you would need to write the logic to handle the messages synchronously. Perhaps emit messages into a queue, and process off of the queue?