reprocess maprstream messages using spring integration kafka - spring-integration

This is related to the thread and I am using spring-integration-kafka 2.0 to consume the messages from mapr stream topics.
I am facing difficulties to use the KafkaConsumer feature - reprocess maprstream messages - using offset and topic partitions.
If I can integrate seek feature I will be able to reprocess the messages based on offset value.
Can someone please help me to integrate the KafkaConsumer features seek, seekToBegining, seekToEnd in spring integration Kafka? The current consumer configuration is mentioned below:
<int-kafka:message-driven-channel-adapter
id="kafkaListener"
listener-container="container1"
auto-startup="true"
phase="100"
send-timeout="5000"
channel="inputFromStream"
error-channel="errorChannel" />
<bean id="container1" class="org.springframework.kafka.listener.KafkaMessageListenerContainer">
<constructor-arg>
<bean class="org.springframework.kafka.core.DefaultKafkaConsumerFactory">
<constructor-arg>
<map>
<entry key="bootstrap.servers" value="localhost:9092"/>
<entry key="group.id" value="siTestGroup1"/>
<entry key="enable.auto.commit" value="true"/>
<entry key="auto.commit.interval.ms" value="1000"/>
<entry key="auto.offset.reset" value="earliest" />
<entry key="max.partition.fetch.bytes" value="3145728"/>
<entry key="key.deserializer" value="org.apache.kafka.common.serialization.StringDeserializer"/>
<entry key="value.deserializer" value="org.apache.kafka.common.serialization.StringDeserializer"/>
</map>
</constructor-arg>
</bean>
</constructor-arg>
<constructor-arg>
<bean class="org.springframework.kafka.listener.config.ContainerProperties">
<constructor-arg name="topics" value="${maprstream.topicname}" />
</bean>
</constructor-arg>
</bean>

Use a ConsumerAwareRebalanceListener - this is how Spring Cloud Stream does it...
final AtomicBoolean initialAssignment = new AtomicBoolean(true);
if (!"earliest".equals(resetTo) && "!latest".equals(resetTo)) {
logger.warn("no (or unknown) " + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG +
" property cannot reset");
resetOffsets = false;
}
if (groupManagement && resetOffsets) {
containerProperties.setConsumerRebalanceListener(new ConsumerAwareRebalanceListener() {
#Override
public void onPartitionsRevokedBeforeCommit(Consumer<?, ?> consumer, Collection<TopicPartition> tps) {
// no op
}
#Override
public void onPartitionsRevokedAfterCommit(Consumer<?, ?> consumer, Collection<TopicPartition> tps) {
// no op
}
#Override
public void onPartitionsAssigned(Consumer<?, ?> consumer, Collection<TopicPartition> tps) {
if (initialAssignment.getAndSet(false)) {
if ("earliest".equals(resetTo)) {
consumer.seekToBeginning(tps);
}
else if ("latest".equals(resetTo)) {
consumer.seekToEnd(tps);
}
}
}
});
}
else if (resetOffsets) {
Arrays.stream(containerProperties.getTopicPartitions())
.map(tpio -> new TopicPartitionInitialOffset(tpio.topic(), tpio.partition(),
// SK GH-599 "earliest".equals(resetTo) ? SeekPosition.BEGINNING : SeekPosition.END))
"earliest".equals(resetTo) ? 0L : Long.MAX_VALUE))
.collect(Collectors.toList()).toArray(containerProperties.getTopicPartitions());
}

Related

MQ Listener Performance

I am building Docker Containers which have a war file running on Jetty, and I have been alternating a few settings to see if performance improves but nothing so far. Per container it has been achieving 7 tps.
The settings are
<bean id="cachingConnectionFactory" class="org.springframework.jms.connection.CachingConnectionFactory">
<property name="targetConnectionFactory" ref="MQConnectionFactory" />
<property name="sessionCacheSize" value="10"/>
</bean>
<bean id="requestQueue" class="com.ibm.mq.jms.MQQueue">
<constructor-arg index="0" value="${queuemanager}"/>
<constructor-arg index="1" value="${incoming.queue}"/>
</bean>
<integration:poller id="poller" default="true" fixed-delay="1000" error-channel="errorChannel"/>
How can I improve the number of threads processing over here?
Also, my connection factory details are as shown below
#Bean(name="DefaultJmsListenerContainerFactory")
public DefaultJmsListenerContainerFactory provideJmsListenerContainerFactory(PlatformTransactionManager transactionManager) {
DefaultJmsListenerContainerFactory factory = new DefaultJmsListenerContainerFactory();
factory.setConnectionFactory(connectionFactory());
factory.setTransactionManager(transactionManager);
factory.setConcurrency(jmsConcurrency);
factory.setCacheLevel(jmsCacheLevel);
factory.setSessionAcknowledgeMode(Session.CLIENT_ACKNOWLEDGE);
factory.setSessionTransacted(true);
return factory;
}
#Bean(name = "txManager")
public PlatformTransactionManager provideTransactionManager() {
return new JmsTransactionManager(connectionFactory());
}
#Bean(name = "JmsTemplate")
public JmsTemplate provideJmsTemplate() {
JmsTemplate jmsTemplate = new JmsTemplate(connectionFactory());
jmsTemplate.setReceiveTimeout(Long.parseLong(env.getRequiredProperty(RECEIVE_TIMEOUT)));
return jmsTemplate;
}
#Bean(name="MQConnectionFactory")
public ConnectionFactory connectionFactory() {
if (factory == null) {
factory = new MQXAConnectionFactory();
try {
factory.setHostName(env.getRequiredProperty(HOST));
factory.setPort(Integer.parseInt(env.getRequiredProperty(PORT)));
factory.setQueueManager(env.getRequiredProperty(QUEUE_MANAGER));
factory.setChannel(env.getRequiredProperty(CHANNEL));
factory.setTransportType(WMQConstants.WMQ_CM_CLIENT);
} catch (JMSException e) {
throw new RuntimeException(e);
}
}
return factory;
}
The initial setting for the concurrency was '1-2' and I changed that to '10-15'. Did not affect performance.
The jmsCache was set to 3 (Consumer cache), but no change there yet either.
Any help is much appreciated.
Cheers
Kris
Answering my own post here. What we found out was that the problem was actually with our Database pooling not setup correctly in the first place.
But in order to increase the Listener count, I had to change my Spring integration adapter settings
<jms:message-driven-channel-adapter id="jmsIn"
destination="requestQueue"
channel="inputJsonConversionChannel"
connection-factory="cachingConnectionFactory"
error-channel="errorChannel"
concurrent-consumers="${jms_adapter_concurrent_consumers}" />
Only when the concurrent-consumers is varied, does the number of listeners on the queue increase.

PropertiesPersistingMetadataStore not writing to file

I am using SftpSimplePatternFileListFilter and SftpPersistentAcceptOnceFileListFilter along with metadata store. But I noticed that it is not flushing the entries to file. I never show flush() method being called from PropertiesPersistingMetadataStore which ultimately invokes saveMetaData() method.
Here is my config looks like
<bean id="compositeFilter" class="org.springframework.integration.file.filters.CompositeFileListFilter">
<constructor-arg>
<list>
<bean class="org.springframework.integration.sftp.filters.SftpSimplePatternFileListFilter">
<constructor-arg value="*.txt" />
</bean>
<bean class="org.springframework.integration.sftp.filters.SftpPersistentAcceptOnceFileListFilter">
<constructor-arg name="store" ref="metadataStore"/>
<constructor-arg value="myapp"/>
</bean>
</list>
</constructor-arg>
</bean>
<bean name="metadataStore" class="org.springframework.integration.metadata.PropertiesPersistingMetadataStore">
<property name="baseDirectory" value="/tmp/"/>
</bean>
By default PropertiesPersistingMetadataStore flushes to the file on applicationContext destroy:
#Override
public void close() throws IOException {
flush();
}
#Override
public void flush() {
saveMetadata();
}
#Override
public void destroy() throws Exception {
flush();
}
Starting with 4.1.2 you can invoke flush() manually at runtime.
E.g. periodically with <task:sheduled-tasks> or with some <int:outbound-channel-adapter>.
Feel free to ask for more information!

Spring integration message-store rolled back before error handler

I need to handle certain error conditions within my spring integration flow. My flow is using a message store and setting the error channel on the poller. I had thought that if I handled the message in the error handler that the rollback would not occur, but the messageStore remove (delete) is being rolled back before the error flow is even executed.
Here is a pseudo-flow that duplicates my issue.
<int:channel id="rollbackTestInput" >
<int:queue message-store="messageStore"/>
</int:channel>
<int:bridge input-channel="rollbackTestInput" output-channel="createException" >
<int:poller fixed-rate="50"
error-channel="myErrorChannel">
<int:transactional />
</int:poller>
</int:bridge>
<int:transformer input-channel="createException" output-channel="infoLogger"
expression="T(GarbageToForceException).doesNotExist()" />
<int:channel id="myErrorChannel">
<int:queue/>
</int:channel>
<!-- JDBC Message Store -->
<bean id="messageStore" class="org.springframework.integration.jdbc.JdbcMessageStore">
<property name="dataSource">
<ref bean="dataSource" />
</property>
</bean>
<bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager" >
<property name="dataSource" ref="dataSource"/>
This flow will result in an infinite rollback/poll loop. How can I handle the error and not rollback?
That's correct behavior. The error-channel logic is accepted around the TX advice for the polling task.
The code looks like:
#Override
public void run() {
taskExecutor.execute(new Runnable() {
#Override
public void run() {
int count = 0;
while (initialized && (maxMessagesPerPoll <= 0 || count < maxMessagesPerPoll)) {
try {
if (!pollingTask.call()) {
break;
}
count++;
}
catch (Exception e) {
if (e instanceof RuntimeException) {
throw (RuntimeException) e;
}
else {
throw new MessageHandlingException(new ErrorMessage(e), e);
}
}
}
}
});
}
Where TX Advice is on the pollingTask.call(), but error handling is done from the taskExecutor:
this.taskExecutor = new ErrorHandlingTaskExecutor(this.taskExecutor, this.errorHandler);
Where your error-channel is configured on that errorHandler as MessagePublishingErrorHandler.
To reach your requirements you can try to follow with synchronization-factory on the <poller>:
<int:transaction-synchronization-factory id="txSyncFactory">
<int:after-rollback channel="myErrorChannel" />
</int:transaction-synchronization-factory>
Or supply your <int:transformer> with <request-handler-advice-chain>:
<int:request-handler-advice-chain>
<bean class="org.springframework.integration.handler.advice.ExpressionEvaluatingRequestHandlerAdvice">
<property name="onFailureExpression" value="#exception" />
<property name="failureChannel" value="myErrorChannel" />
<property name="trapException" value="true" />
</bean>
</int:request-handler-advice-chain>
I have found a different solution that meets my requirements and is less invasive.
Instead of using transactional on the poller, I can use an advice chain that that specifies which exceptions I should rollback and which I shouldn't. For my case, I do not want to rollback most exceptions. So I rollback Throwable and list any specific exceptions I want to rollback.
<int:bridge input-channel="rollbackTestInput"
output-channel="createException">
<int:poller fixed-rate="50"
error-channel="myErrorChannel">
<int:advice-chain>
<int:ref bean="txAdvice"/>
</int:advice-chain>
</int:poller>
</int:bridge>
<tx:advice id="txAdvice" transaction-manager="transactionManager">
<tx:attributes>
<tx:method name="*" rollback-for="javax.jms.JMSException"
no-rollback-for="java.lang.Throwable" />
</tx:attributes>
</tx:advice>

Resttemplate : HttpMessageNotWritableException: No serializer found for class java.io.ByteArrayInputStream

Using Spring Integration I upload a file via HTTP POST and route it to a service activator.
In the service activator, I make a call using RestTemplate to another server where to dump the file but I can't figure out why I get the following error for the following code:
What I don't understand is why I get the exception below when I call RestTemplate.exchange()
at pdi.integration.MultipartReceiver.printMultiPartContent(MultipartReceiver.java:41)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
xml-config
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-http="http://www.springframework.org/schema/integration/http"
xsi:schemaLocation="http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration/http http://www.springframework.org/schema/integration/http/spring-integration-http.xsd">
<bean id="byteArrayHttpMessageConverter"
class="org.springframework.http.converter.ByteArrayHttpMessageConverter">
</bean>
<bean id="formHttpMessageConverter"
class="org.springframework.http.converter.FormHttpMessageConverter">
</bean>
<bean id="multipartResolver"
class="org.springframework.web.multipart.commons.CommonsMultipartResolver"/>
<bean id="headerMapper" class="org.springframework.integration.http.support.DefaultHttpHeaderMapper">
<property name="inboundHeaderNames" value="*"/>
<property name="outboundHeaderNames" value="*"/>
<property name="userDefinedHeaderPrefix" value=""/>
</bean>
<int:channel id="http.frontend.rx"/>
<int:channel id="http.frontend.tx"/>
<int:channel id="http.backend.mysql.rx"/>
<int:channel id="http.backend.mysql.tx"/>
<int:channel id="http.backend.mongo.rx"/>
<int:channel id="http.backend.mongo.tx"/>
<int:channel id="http.backend.file.tx"/>
<int-http:inbound-gateway
id="frontEndToMySQLXX"
request-channel="http.frontend.rx"
reply-channel="http.frontend.tx"
header-mapper="headerMapper"
path="/gateway"
supported-methods="GET,POST,PUT,DELETE"/>
<int:router id="frontEndRouter" input-channel="http.frontend.rx" expression="headers.service">
<int:mapping value="json" channel="http.backend.mysql.tx" />
<int:mapping value="file" channel="http.backend.mongo.tx" />
<int:mapping value="upload" channel="http.backend.file.tx" />
</int:router>
<!-- removed : message-converters="formHttpMessageConverter,byteArrayHttpMessageConverter" -->
<int-http:outbound-gateway
id="toMongoDB"
request-channel="http.backend.mongo.tx"
reply-channel="http.backend.mongo.rx"
url="http://localhost:5050/api/{path}"
http-method-expression="headers.http_requestMethod"
header-mapper="headerMapper"
expected-response-type="byte[]">
<int-http:uri-variable name="path" expression="headers['urlpath']"/>
</int-http:outbound-gateway>
<int-http:outbound-gateway
id="toMySQLDB"
request-channel="http.backend.mysql.tx"
reply-channel="http.backend.mysql.rx"
url="http://localhost:7070/api/{path}"
http-method-expression="headers.http_requestMethod"
expected-response-type="java.lang.String"
charset="UTF-8">
<int-http:uri-variable name="path" expression="headers['urlpath']"/>
</int-http:outbound-gateway>
<int:service-activator
id="MySQLToFrontEnd"
input-channel="http.backend.mysql.rx"
output-channel="http.frontend.tx"
ref="messageService"
method="printContent">
</int:service-activator>
<int:service-activator
id="MongoToFrontEnd"
input-channel="http.backend.file.tx"
output-channel="http.frontend.tx"
ref="multipartReceiver"
method="printMultiPartContent">
</int:service-activator>
</beans>
bean used by service activator
#Component
public class MultipartReceiver {
public void printMultiPartContent(LinkedMultiValueMap<String, Object> multipartRequest){
System.out.println("### Successfully received multipart request ###");
for (String elementName : multipartRequest.keySet()) {
if (elementName.equals("file")){
System.out.println("\t" + elementName + " - as UploadedMultipartFile: " +
((UploadedMultipartFile) multipartRequest
.getFirst("file")).getOriginalFilename());
}
}
RestTemplate template = new RestTemplate();
String uri = "http://localhost:5050/api/upload";
MultiValueMap map = new LinkedMultiValueMap<String,Object>();
map.add("file", multipartRequest.getFirst("file"));
HttpHeaders headers = new HttpHeaders();
headers.set("Content-Type", "multipart/form-data");
HttpEntity request = new HttpEntity(map, headers);
ResponseEntity<byte[]> response = template.exchange(uri, HttpMethod.POST, request, byte[].class);
}
}
stacktrace :: http://pastebin.com/5Wa9VaRb
working code ::
public void printMultiPartContent(LinkedMultiValueMap<String, Object> multipartRequest) throws IOException {
final String filename = ((UploadedMultipartFile) multipartRequest.getFirst("file")).getOriginalFilename();
RestTemplate template = new RestTemplate();
MultiValueMap<String, Object> multipartMap = new LinkedMultiValueMap<String, Object>();
multipartMap.add("name", filename);
multipartMap.add("filename", filename);
byte[] bytes = ((UploadedMultipartFile) multipartRequest.getFirst("file")).getBytes();
ByteArrayResource contentsAsResource = new ByteArrayResource(bytes){
public String getFilename(){
return filename;
}
};
multipartMap.add("file", contentsAsResource);
String result = template.postForObject("http://localhost:5050/api/upload", multipartMap, String.class);
System.out.println(result);
}
The problem is you are passing the complete UploadedMultipartFile object to the RestTemplate. Jackson is trying to serialize the object, including the inputStream property, which it cannot.
It appears you need to extract the file contents
byte[] bytes ((UploadedMultipartFile) multipartRequest.getFirst("file")).getBytes();
And set the content type to the UploadedMultipartFile.getContentType().

RESTEasy + Spring + Jackson + Jaxb

I am trying to serialize JAXB annotated class with Jackson using RESTEasy. By default ResteasyJacksonProvider is configured to use JACKSON annotation only. Is there a way to configure ResteasyJacksonProvider to use JAXB annotation using spring? There are couple of programmatic ways but would prefer if there is some spring configuration.
Couple of ways I am thinking of
Use ContextResolver for ObjectMapper type to return ObjectMapper configured to use JaxbAnnotationIntrospector instead of JacksonAnnotationIntrospector.
Extend ResteasyJacksonProvider and pass JAXB annotation during construction.
Any other way?
Well the first option of using ContextResolver works but I still think there should be an easier way to do this just by some configuration.
You can get this from the configuration only, no need to program anything special.
Here's how to:
First set your configuration right, I use Jackson + JAXB, both set under ContentNegotiatingViewResolver bean:
<bean class="org.springframework.web.servlet.view.ContentNegotiatingViewResolver">
<property name="order" value="1"/>
<property name="mediaTypes">
<map>
<entry key="xml" value="application/xml" />
<entry key="json" value="application/json" />
</map>
</property>
<property name="defaultViews">
<list>
<bean class="org.springframework.web.servlet.view.xml.MarshallingView">
<property name="marshaller">
<oxm:jaxb2-marshaller id="marshaller">
<oxm:class-to-be-bound name="com.shay.dashboard.data.structure.page.PageObject" />
<oxm:class-to-be-bound name="com.shay.dashboard.data.structure.tab.TabObject" />
<oxm:class-to-be-bound name="com.shay.dashboard.data.structure.section.SectionObject" />
<oxm:class-to-be-bound name="com.shay.dashboard.data.structure.element.nonembedded.ElementObject"/>
<oxm:class-to-be-bound name="com.shay.dashboard.data.structure.element.embedded.EmbeddedElementObject"/>
<oxm:class-to-be-bound name="com.shay.dashboard.data.structure.chart.common.ChartManager"/>
</oxm:jaxb2-marshaller>
</property>
</bean>
<bean class="org.springframework.web.servlet.view.json.MappingJacksonJsonView"/>
</list>
</property>
</bean>
Notice that under the marshaller I set the oxm:class-to-be-bound - those are the classes to be bound by JAXB.
Now for the module, I used ordinary annotation package (javax.xml.bind.annotation), non marshaller specific. Jackson Json and JAXB both know how to read it.
For example:
#XmlAccessorType(XmlAccessType.NONE)
#XmlRootElement(name="page")
public class PageObject implements ComponentTypeObject{
#XmlAttribute(name="name")
private String name;
#XmlAttribute(name="id",required=true)
private String id;
#XmlElements({#XmlElement(name="tab", type=TabXmlAdapter.class)})
private List<TabXmlAdapter> tabRef;
Finally the controller for your MVC needs to return a model and view:
#RequestMapping(value="/get_page", method = RequestMethod.GET)
public ModelAndView initPage()
{
ModelAndView mav = null;
try
{
PageObject myPage = (PageObject) Utilities.getUtilities().loadObjectFromFile(XmlComponentType.page);
mav = new ModelAndView("page","page",myPage);
}
catch (Exception e)
{
e.getMessage();
}
return mav;
}
Now while calling your URL ending with .json you'd get the JSON representation, and with .xml - and XML. Both are translated by the viewer, provided you gave the correct mapping when annotating the module.

Resources