Dynamically Create Kafka Consumers That Send Data Through a Websocket - node.js

Question

I am using kafka-node to read stream data and pass it to my web app with Web Sockets using NodeJS. This works fine if I able to define the kafka producer server and the topic I am interested in, however for my use case the end users will input the kafka producer server and the topic and my NodeJS backend will be responsible to receive that request and to create the appropriate kafka/websocket connections.
My idea was the following:
Create a rest API to which the web app could send requests to in order to create a new kafka consumer/web socket connection (/registerTopic)
Save the new kafka consumers in a global array when I create a new kafka consumer so that I can later pause or resume the stream with another rest API call (/pauseTopic and /resumeTopic)
I ran into problems trying to move the WebSocket code into /registerTopic...Whenever I do this everything acts very strangely and I suddenly get 1000x messages at once and then 40-50x messages every second even though the kafka producer is only sending 1 message per second. Any ideas on how I can get this working?
const express = require("express");
const app = express();
const cors = require('cors');
const bodyParser = require('body-parser');
const fs = require('fs');
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
var WebSocketServer = require('websocket').server;
var http = require('http');
var https = require('https');
var kafka = require('kafka-node');
var topics = [];
var privateKey = fs.readFileSync('PATH', 'utf8');
var certificate = fs.readFileSync('PATH', 'utf8');
var credentials = { key: privateKey, cert: certificate };
var consumers = new Set();
// This was working without any issues before I tried to make this dynamic!
/* var Consumer = kafka.Consumer,
client = new kafka.KafkaClient('localhost:9092'),
consumer = new Consumer(
client, [{ topic: 'numtest', partition: 0 }], { autoCommit: false }); */
var server = http.createServer(function (request, response) {
console.log(' Request recieved : ' + request.url);
response.writeHead(404);
response.end();
});
server.listen(8080, function () {
console.log('Listening on port : 8080');
});
webSocketServer = new WebSocketServer({
httpServer: server,
autoAcceptConnections: false
});
function iSOriginAllowed(origin) {
return true;
}
// This was working without any issues before I tried to make this dynamic!
/* webSocketServer.on('request', function (request) {
if (!iSOriginAllowed(request.origin)) {
request.reject();
console.log(' Connection from : ' + request.origin + ' rejected.');
return;
}
let connection = request.accept('echo-protocol', request.origin);
console.log(' Connection accepted : ' + request.origin);
connection.on('message', function (message) {
if (message.type === 'utf8') {
console.log('Received Message: ' + message.utf8Data);
}
});
consumer.on('message', function (message) {
console.log('msg');
connection.sendUTF(message.value);
});
connection.on('close', function (reasonCode, description) {
console.log('Connection ' + connection.remoteAddress + ' disconnected.');
});
}); */
var httpsServer = http.createServer(credentials, app);
httpsServer.listen(3000, () => {
console.log("Server running on port 3000");
});
app.get("/getTopics", (req, res, next) => {
res.json(topics);
});
app.post("/registerTopic", (req, res) => {
try {
var client = new kafka.KafkaClient(req.body.host);
var Consumer = kafka.Consumer;
consumer = new Consumer(
client, [{ topic: req.body.topic, partition: 0 }], { autoCommit: false });
let consumerExists = false;
for (let c = 0; c < [...consumers].length; c++) {
if ([...consumers][c].topic == req.body.topic && [...consumers][c].sessionId == req.body.sessionId) {
consumerExists = true;
}
}
if (!consumerExists) {
consumers.add({ 'topic': req.body.topic, 'sessionId': req.body.sessionId, 'consumer': consumer });
}
client.loadMetadataForTopics([], function (error, results) {
Object.keys(results[1].metadata).forEach(function (key) {
var value = results[1].metadata[key];
if (!value['0'].topic.includes('__') && !value['0'].topic.includes('offset')) {
topics.push({ 'producer': req.body.host, 'topic': value['0'].topic });
}
});
});
webSocketServer.on('request', function (request) {
if (!iSOriginAllowed(request.origin)) {
request.reject();
console.log(' Connection from : ' + request.origin + ' rejected.');
return;
}
let connection = request.accept('echo-protocol', request.origin);
console.log(' Connection accepted : ' + request.origin);
connection.on('message', function (message) {
if (message.type === 'utf8') {
console.log('Received Message: ' + message.utf8Data);
}
});
consumer.on('message', function (message) {
console.log('msg');
connection.sendUTF(message.value);
});
connection.on('close', function (reasonCode, description) {
console.log('Connection ' + connection.remoteAddress + ' disconnected.');
});
});
res.json("Working");
} catch (error) {
console.error(error);
res.status(400).send('Unable to register new topic')
}
});
app.post("/pauseTopic", (req, res) => {
try {
console.log(req.body);
let filteredConsumer = [...consumers].filter(function (item) {
console.log(req.body.topic, item.sessionId);
if (item.topic == req.body.topic && item.sessionId == req.body.sessionId) {
return c;
}
});
console.log(filteredConsumer);
//filteredConsumer[0].consumer.pause();
res.json("Working");
} catch (error) {
console.error(error);
res.status(400).send('Unable to register new topic')
}
});
app.post("/resumeTopic", (req, res) => {
try {
let filteredConsumer = [...consumers].filter(function (item) {
if (item.topic == req.body.topic && item.sessionId == req.body.sessionId) {
return item;
}
});
filteredConsumer[0].consumer.resume();
res.json("Working");
} catch (error) {
console.error(error);
res.status(400).send('Unable to register new topic')
}
});