node-rdkafka producer not connecting/producing in docker alpine but running perfectly locally - node.js

Question

I have the following setup
Dockerfile
FROM node:alpine
WORKDIR /usr/src/app
RUN apk --no-cache --update add python3 py3-pip alpine-sdk bash librdkafka openssl-dev
COPY package*.json ./
RUN npm install
COPY . .
EXPOSE 5004
CMD ["node", "app.js"]
A bills.js file
import fs from "fs";
import readline from "readline";
function readAllLines(path) {
return new Promise((resolve, reject) => {
// Test file access directly, so that we can fail fast.
// Otherwise, an ENOENT is thrown in the global scope by the readline internals.
try {
fs.accessSync(path, fs.constants.R_OK);
} catch (err) {
reject(err);
}
let lines = [];
const reader = readline.createInterface({
input: fs.createReadStream(path),
crlfDelay: Infinity
});
reader
.on('line', (line) => lines.push(line))
.on('close', () => resolve(lines));
});
}
const configFromPath = async function configFromPath(path) {
const lines = await readAllLines(path);
return lines
.filter((line) => !/^\s*?#/.test(line))
.map((line) => line
.split('=')
.map((s) => s.trim()))
.reduce((config, [k, v]) => {
config[k] = v;
return config;
}, {});
};
// Copy
// Next, we are going to create the producer application by pasting the following code into a file named producer.js:
import Kafka from "node-rdkafka";
function createConfigMap(config) {
if (config.hasOwnProperty('security.protocol')) {
return {
'bootstrap.servers': config['bootstrap.servers'],
'sasl.username': config['sasl.username'],
'sasl.password': config['sasl.password'],
'security.protocol': config['security.protocol'],
'sasl.mechanisms': config['sasl.mechanisms'],
'dr_msg_cb': true,
'request.required.acks': 1
}
} else {
return {
'bootstrap.servers': config['bootstrap.servers'],
'dr_msg_cb': true
}
}
}
function createProducer(config, onDeliveryReport) {
const producer = new Kafka.Producer(createConfigMap(config));
return new Promise((resolve, reject) => {
producer
.on('ready', () => resolve(producer))
.on('delivery-report', onDeliveryReport)
.on('event.error', (err) => {
console.warn('event.error', err);
reject(err);
});
producer.connect();
});
}
async function produceExample(value,key) {
let configPath ="kafka.properties";
const config = await configFromPath(configPath);
value = Buffer.from(JSON.stringify(value));
let topic = "bill";
const producer = await createProducer(config, (err, report) => {
if (err) {
console.warn('Error producing', err)
} else {
const {topic, key, value} = report;
let k = key.toString().padEnd(10, ' ');
console.log(`Produced event to topic ${topic}: key = ${k} value = ${value}`);
}
});
producer.produce(topic, -1, value, key);
producer.flush(10000, () => {
producer.disconnect();
});
}
Here's my config file
bootstrap.servers=xxxx.gcp.confluent.cloud:9092
security.protocol=SASL_SSL
sasl.mechanisms=PLAIN
sasl.username=xxxxxx
sasl.password=xxxxxx
# Best practice for higher availability in librdkafka clients prior to 1.7
session.timeout.ms=45000
If I debug this code and write a console.log just below producer.connect() it's working but not comming down to
producer.produce(topic, -1, value, key);
I have tested this code without dockerizing and it's working fine.
I also see some errors while building node-rdkafka when Dockerfile run npm install
I think the node-rdfkafka is not building properly and that's causing the issue of stuck behavior.
Here are some images of the build error I suspect
Many of these types of errors
Edit:
I'm now getting this error now even after making librdkafka from the source. Error