Node: Sending Protobuf message to Kafka Error - node.js

Question

I am trying to use the HDFS kafka connector to send protobuf messages from kafka to the HDFS. My connector config looks like the following
{
"name": "hdfs3-connector-test",
"config": {
"connector.class": "io.confluent.connect.hdfs3.Hdfs3SinkConnector",
"tasks.max": "1",
"topics": "test-topic",
"hdfs.url": "hdfs://10.8.0.1:9000",
"flush.size": "3",
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
"value.converter": "io.confluent.connect.protobuf.ProtobufConverter",
"value.converter.schema.registry.url":"http://10.8.0.1:8081",
"confluent.topic.bootstrap.servers": "10.8.0.1:9092",
"confluent.topic.replication.factor": "1"
}
}
In order to test this, I am trying to send protobuf serialized messages in a small node application. Here are my files:
// data.proto
syntax = "proto3";
package awesomepackage;
message SearchRequest {
string query = 1;
int32 page = 2;
}
and my node app
const { Kafka } = require('kafkajs')
const protobuf = require('protobufjs')
const kafka = new Kafka({
clientId: 'my-app',
brokers: ['10.8.0.1:9092']
})
const producer = kafka.producer()
const run = async () => {
await producer.connect()
protobuf.load('data.proto', async (err, root) => {
console.log("TESTING")
console.log(err)
let SearchRequest = root.lookupType('awesomepackage.SearchRequest')
let payload = {query: "test", page: 2}
var errMsg = SearchRequest.verify(payload);
console.log(errMsg)
let msg = SearchRequest.create(payload)
var buffer = SearchRequest.encode(msg).finish();
console.log(buffer)
await producer.send({
topic: 'test-topic',
messages: [
{key: 'key1', value: buffer}
]
})
})
}
run()
However, when I run this I get the following errors:
Failed to deserialize data for topic test-topic to Protobuf
Caused by: org.apache.kafka.common.errors.SerializationException: Error deserializing Protobuf message for id -1
Caused by: org.apache.kafka.common.errors.SerializationException: Unknown magic byte!
How do I fix this? My guess is that my protobuf schema is not registered in the kafka schema registry, but I am unsure. If this is the case, is there a way to send the schema to the registry from node?

Answer 1

io.confluent.connect.protobuf.ProtobufConverter requires the Schema Registry, not plain serialized Protobuf. In other words, you're missing the Schema Registry part (or a manual byte creation of a "wrapped" Proto message) in the Node code
Refer Wire Format - Confluent
If you would like to not use the Schema Registry, you can use the BlueApron Protobuf Converter, but seems like you are using one, so best to go with the Confluent converter