I am publishing a MapMessage on IBM MQ and using SI i am receiving that message (using jms:inbound-gateway). I want to transform this Map into an object so i am using map-to-object-transformer. When i receive a message and transformer tries to transform this map into an object i am getting ClassCastException.
MapMessage Publisher
public void sendMapMessage(){
jmsTemplate102.send(sendQueue,
new MessageCreator() {
public Message createMessage(Session session) throws JMSException {
return getEventMap(session);
}
public Message getEventMap(Session session){
MapMessage mapMsg = null;
try {
mapMsg = session.createMapMessage();
mapMsg.setStringProperty("batchId", "123987");
mapMsg.setStringProperty("eventType", "ABCD");
mapMsg.setStringProperty("client", "CORP");
} catch (JMSException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return mapMsg;
}
});
}
JMS Inbound Gateway
<int-jms:inbound-gateway id="jmsInGateway" request-destination="receiverQueue"
request-channel="events"
concurrent-consumers="1"
max-concurrent-consumers="4"
acknowledge="auto"
/>
<int:channel id="etlEvents"/>
<int:map-to-object-transformer input-channel="etlEvents" output-channel="etlEvents" ref="etlEvent"/>
Exception
JMS Message class: jms_text
JMSType: null
JMSDeliveryMode: 2
JMSExpiration: 0
JMSPriority: 4
JMSMessageID: ID:414d5120414344534430513720202020537502dd20130f02
JMSTimestamp: 1404142195240
JMSCorrelationID:null
JMSDestination: null
JMSReplyTo: null
JMSRedelivered: false
JMS_IBM_PutDate:20140630
JMSXAppID:Websphere MQ Client for Java
JMS_IBM_Format:MQSTR
JMS_IBM_PutApplType:28
JMS_IBM_MsgType:8
JMSXUserID:mqcls1
JMS_IBM_PutTime:15295524
JMSXDeliveryCount:1
<map></map>] to integration Message payload [<map></map>]
2014-06-30 10:48:13,356 DEBUG [org.springframework.jms.listener.DefaultMessageListenerContainer#0-1] channel.DirectChannel - preSend on channel 'etlEvents', message: [Payload String content=<map></map>][Headers={JMS_IBM_Format=MQSTR , errorChannel=org.springframework.messaging.core.GenericMessagingTemplate$TemporaryReplyChannel#581bce9a, jms_timestamp=1404142195240, replyChannel=org.springframework.messaging.core.GenericMessagingTemplate$TemporaryReplyChannel#581bce9a, JMSXUserID=mqcls1 , JMS_IBM_PutApplType=28, JMS_IBM_MsgType=8, JMS_IBM_PutDate=20140630, jms_messageId=ID:414d5120414344534430513720202020537502dd20130f02, JMS_IBM_PutTime=15295524, JMSXDeliveryCount=1, jms_redelivered=false, priority=4, JMSXAppID=Websphere MQ Client for Java, id=f8aacfea-e655-e19d-1b7f-fd99389bf095, timestamp=1404139693356}]
2014-06-30 10:48:13,356 DEBUG [org.springframework.jms.listener.DefaultMessageListenerContainer#0-1] transformer.MessageTransformingHandler - org.springframework.integration.transformer.MessageTransformingHandler#0 received message: [Payload String content=<map></map>][Headers={JMS_IBM_Format=MQSTR , errorChannel=org.springframework.messaging.core.GenericMessagingTemplate$TemporaryReplyChannel#581bce9a, jms_timestamp=1404142195240, replyChannel=org.springframework.messaging.core.GenericMessagingTemplate$TemporaryReplyChannel#581bce9a, JMSXUserID=mqcls1 , JMS_IBM_PutApplType=28, JMS_IBM_MsgType=8, JMS_IBM_PutDate=20140630, jms_messageId=ID:414d5120414344534430513720202020537502dd20130f02, JMS_IBM_PutTime=15295524, JMSXDeliveryCount=1, jms_redelivered=false, priority=4, JMSXAppID=Websphere MQ Client for Java, id=f8aacfea-e655-e19d-1b7f-fd99389bf095, timestamp=1404139693356}]
2014-06-30 10:48:13,356 WARN [org.springframework.jms.listener.DefaultMessageListenerContainer#0-1] jms.ChannelPublishingJmsMessageListener$GatewayDelegate - failure occurred in gateway sendAndReceive
org.springframework.integration.transformer.MessageTransformationException: failed to transform message
at org.springframework.integration.transformer.AbstractTransformer.transform(AbstractTransformer.java:44)
at org.springframework.integration.transformer.MessageTransformingHandler.handleRequestMessage(MessageTransformingHandler.java:68)
at org.springframework.integration.handler.AbstractReplyProducingMessageHandler.handleMessageInternal(AbstractReplyProducingMessageHandler.java:170)
at org.springframework.integration.handler.AbstractMessageHandler.handleMessage(AbstractMessageHandler.java:78)
at org.springframework.integration.dispatcher.AbstractDispatcher.tryOptimizedDispatch(AbstractDispatcher.java:116)
at org.springframework.integration.dispatcher.UnicastingDispatcher.doDispatch(UnicastingDispatcher.java:101)
at org.springframework.integration.dispatcher.UnicastingDispatcher.dispatch(UnicastingDispatcher.java:97)
at org.springframework.integration.channel.AbstractSubscribableChannel.doSend(AbstractSubscribableChannel.java:77)
at org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:255)
at org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:114)
at org.springframework.messaging.core.GenericMessagingTemplate.doSendAndReceive(GenericMessagingTemplate.java:154)
at org.springframework.messaging.core.GenericMessagingTemplate.doSendAndReceive(GenericMessagingTemplate.java:44)
at org.springframework.messaging.core.AbstractMessagingTemplate.sendAndReceive(AbstractMessagingTemplate.java:75)
at org.springframework.integration.gateway.MessagingGatewaySupport.doSendAndReceive(MessagingGatewaySupport.java:250)
at org.springframework.integration.gateway.MessagingGatewaySupport.sendAndReceiveMessage(MessagingGatewaySupport.java:224)
at org.springframework.integration.jms.ChannelPublishingJmsMessageListener$GatewayDelegate.sendAndReceiveMessage(ChannelPublishingJmsMessageListener.java:485)
at org.springframework.integration.jms.ChannelPublishingJmsMessageListener.onMessage(ChannelPublishingJmsMessageListener.java:330)
at org.springframework.jms.listener.AbstractMessageListenerContainer.doInvokeListener(AbstractMessageListenerContainer.java:537)
at org.springframework.jms.listener.AbstractMessageListenerContainer.invokeListener(AbstractMessageListenerContainer.java:497)
at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:468)
at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325)
at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1102)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1094)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:991)
at java.lang.Thread.run(Unknown Source)
Caused by: java.lang.ClassCastException: java.lang.String cannot be cast to java.util.Map
at org.springframework.integration.transformer.MapToObjectTransformer.transformPayload(MapToObjectTransformer.java:41)
at org.springframework.integration.transformer.AbstractPayloadTransformer.doTransform(AbstractPayloadTransformer.java:33)
at org.springframework.integration.transformer.AbstractTransformer.transform(AbstractTransformer.java:33)
... 25 more
2014-06-30 10:48:13,387 WARN [org.springframework.jms.listener.DefaultMessageListenerContainer#0-1] listener.DefaultMessageListenerContainer - Execution of JMS message listener failed, and no ErrorHandler has been set.
org.springframework.integration.transformer.MessageTransformationException: failed to transform message
at org.springframework.integration.transformer.AbstractTransformer.transform(AbstractTransformer.java:44)
at org.springframework.integration.transformer.MessageTransformingHandler.handleRequestMessage(MessageTransformingHandler.java:68)
at org.springframework.integration.handler.AbstractReplyProducingMessageHandler.handleMessageInternal(AbstractReplyProducingMessageHandler.java:170)
at org.springframework.integration.handler.AbstractMessageHandler.handleMessage(AbstractMessageHandler.java:78)
at org.springframework.integration.dispatcher.AbstractDispatcher.tryOptimizedDispatch(AbstractDispatcher.java:116)
at org.springframework.integration.dispatcher.UnicastingDispatcher.doDispatch(UnicastingDispatcher.java:101)
at org.springframework.integration.dispatcher.UnicastingDispatcher.dispatch(UnicastingDispatcher.java:97)
at org.springframework.integration.channel.AbstractSubscribableChannel.doSend(AbstractSubscribableChannel.java:77)
at org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:255)
at org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:114)
at org.springframework.messaging.core.GenericMessagingTemplate.doSendAndReceive(GenericMessagingTemplate.java:154)
at org.springframework.messaging.core.GenericMessagingTemplate.doSendAndReceive(GenericMessagingTemplate.java:44)
at org.springframework.messaging.core.AbstractMessagingTemplate.sendAndReceive(AbstractMessagingTemplate.java:75)
at org.springframework.integration.gateway.MessagingGatewaySupport.doSendAndReceive(MessagingGatewaySupport.java:250)
at org.springframework.integration.gateway.MessagingGatewaySupport.sendAndReceiveMessage(MessagingGatewaySupport.java:224)
at org.springframework.integration.jms.ChannelPublishingJmsMessageListener$GatewayDelegate.sendAndReceiveMessage(ChannelPublishingJmsMessageListener.java:485)
at org.springframework.integration.jms.ChannelPublishingJmsMessageListener.onMessage(ChannelPublishingJmsMessageListener.java:330)
at org.springframework.jms.listener.AbstractMessageListenerContainer.doInvokeListener(AbstractMessageListenerContainer.java:537)
at org.springframework.jms.listener.AbstractMessageListenerContainer.invokeListener(AbstractMessageListenerContainer.java:497)
at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:468)
at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325)
at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1102)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1094)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:991)
at java.lang.Thread.run(Unknown Source)
Caused by: java.lang.ClassCastException: java.lang.String cannot be cast to java.util.Map
at org.springframework.integration.transformer.MapToObjectTransformer.transformPayload(MapToObjectTransformer.java:41)
at org.springframework.integration.transformer.AbstractPayloadTransformer.doTransform(AbstractPayloadTransformer.java:33)
at org.springframework.integration.transformer.AbstractTransformer.transform(AbstractTransformer.java:33)
... 25 more
Sorry for the bad formatting.
Can anyone please tell me what wrong i am doing here ?
Looks like IBM MQ doesn't support MapMessage. The exaption said that you receive:
JMS Message class: jms_text
And you get just a TextMessage with xml <map></map>:
[Payload String content=<map></map>]
UPDATE
OK: let IBM MQ supports the MapMessage. So, would be great, if you check it in the DEBUG for the ChannelPublishingJmsMessageListener. Or at least switch on the DEBUG logging level for that class and show here the result of its expression:
logger.debug("converted JMS Message [" + jmsMessage + "] to integration Message payload [" + result + "]");
UPDATE 2
I see, sorry. Can't it a wrong message in the queue? However about to clean the MQ queue and try one more time? Because now it is String and it is just <map></map>. As you see there is no your sent content. Actually the framework works well with this issue and can't complain, because your message is wrong.
Related
How do I handle errors thrown in the below SpringIntegrationFlow just by logging the error and continuing with the next record. As well should be able to send an email only once irrespective of 1 or more records failed. Is there something equivalent of Spring Batch's StepListener? Thanks
return IntegrationFlows.from(jdbcMessageSource(), p -> p.poller(pollerSpec()))
.enrichHeaders(Collections.singletonMap(MessageHeaders.ERROR_CHANNEL, appErrorChannel()))
.split()
.channel(c -> c.executor(Executors.newCachedThreadPool()))
.transform(transformer, "transform")
.enrichHeaders(headerEnricherSpec -> headerEnricherSpec.header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE))
.handle(Http.outboundGateway(url)
.httpMethod(HttpMethod.POST)
.expectedResponseType(String.class)
.requestFactory(requestFactory))
.get();
#Bean
public MessageChannel appErrorChannel() {
return new DirectChannel();
}
#Bean("appErrorFlow")
public IntegrationFlow appErrorFlow() {
// #formatter:off
return IntegrationFlows.from(appErrorChannel())
.log(Level.ERROR, message -> " Failed Message " + message.getPayload())
.aggregate(a -> a.correlationStrategy(m -> !m.getHeaders().isEmpty()))
.handle(Http.outboundGateway(mailURL)
.httpMethod(HttpMethod.POST))
.get();
// #formatter:on
}
Exception:
2022-01-13 06:41:31,702 [Worker-1 ] WARN o.s.i.c.MessagePublishingErrorHandler - 005006f2dee5b673 Error message was not delivered.
org.springframework.messaging.MessageDeliveryException: Dispatcher has no subscribers for channel 'unknown.channel.name'.; nested exception is org.springframework.integration.MessageDispatchingException: Dispatcher has no subscribers, failedMessage=ErrorMessage [payload=*******Removed On Purpose*******]
at org.springframework.integration.channel.AbstractSubscribableChannel.doSend(AbstractSubscribableChannel.java:76)
at org.springframework.integration.channel.AbstractMessageChannel.send(AbstractMessageChannel.java:317)
at org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:187)
at org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:166)
at org.springframework.messaging.core.GenericMessagingTemplate.doSend(GenericMessagingTemplate.java:47)
at org.springframework.messaging.core.AbstractMessageSendingTemplate.send(AbstractMessageSendingTemplate.java:109)
at org.springframework.integration.channel.MessagePublishingErrorHandler.handleError(MessagePublishingErrorHandler.java:96)
at org.springframework.integration.util.ErrorHandlingTaskExecutor.lambda$execute$0(ErrorHandlingTaskExecutor.java:60)
at org.springframework.cloud.sleuth.instrument.async.TraceRunnable.run(TraceRunnable.java:64)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: org.springframework.integration.MessageDispatchingException: Dispatcher has no subscribers
at org.springframework.integration.dispatcher.UnicastingDispatcher.doDispatch(UnicastingDispatcher.java:139)
at org.springframework.integration.dispatcher.UnicastingDispatcher.dispatch(UnicastingDispatcher.java:106)
at org.springframework.integration.channel.AbstractSubscribableChannel.doSend(AbstractSubscribableChannel.java:72)
... 11 common frames omitted
I believe the closest equivalent for StepListener is a ChannelInterceptor.
See IntegrationFlowDefinition.intercept() operator to place in between your endpoints in that flow.
However according your splitter configuration, it doesn't seem like you are going to lose anything and errors are just going to be logged. That c.executor() does the trick to shift the work for each record into a thread pool and its error cannot effect the rest of records.
Anyway it sounds like better for you to have a custom error channel be set into headers just before that split() - .enrichHeaders(Collections.singletonMap(MessageHeaders.ERROR_CHANNEL, myErrorChannel())).
Than you have an aggregator subscribed to this channel where you aggregate errors for the batch and emit a message to send an email. This is going to happen really only once per batch or none at all if no errors.
Transformer throws exception for returning null. I'm getting the message payload and doing my business logic in transformer. Then, sending response to fileoutput channel. I've tried using .handle method too instead of transformer, but getting one way message exception.
EDIT
#Bean
IntegrationFlow integrationFlow() {
return IntegrationFlows.from(this.sftpMessageSource()).channel(fileInputChannel()).
handle(service, "callMethod").channel(fileOutputChannel()).
handle(orderOutMessageHandler()).get();
}
EDIT 2
[ERROR] 2020-06-14 14:49:48.053 [task-scheduler-9] LoggingHandler - java.lang.AbstractMethodError: Method org/springframework/integration/sftp/session/SftpSession.getHostPort()Ljava/lang/String; is abstract
at org.springframework.integration.sftp.session.SftpSession.getHostPort(SftpSession.java)
at org.springframework.integration.file.remote.session.CachingSessionFactory$CachedSession.getHostPort(CachingSessionFactory.java:295)
at org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer.copyFileToLocalDirectory(AbstractInboundFileSynchronizer.java:496)
at org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer.copyIfNotNull(AbstractInboundFileSynchronizer.java:400)
at org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer.transferFilesFromRemoteToLocal(AbstractInboundFileSynchronizer.java:386)
at org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer.lambda$synchronizeToLocalDirectory$0(AbstractInboundFileSynchronizer.java:349)
at org.springframework.integration.file.remote.RemoteFileTemplate.execute(RemoteFileTemplate.java:437)
at org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer.synchronizeToLocalDirectory(AbstractInboundFileSynchronizer.java:348)
at org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizingMessageSource.doReceive(AbstractInboundFileSynchronizingMessageSource.java:265)
at org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizingMessageSource.doReceive(AbstractInboundFileSynchronizingMessageSource.java:66)
at org.springframework.integration.endpoint.AbstractFetchLimitingMessageSource.doReceive(AbstractFetchLimitingMessageSource.java:45)
at org.springframework.integration.endpoint.AbstractMessageSource.receive(AbstractMessageSource.java:167)
at org.springframework.integration.endpoint.SourcePollingChannelAdapter.receiveMessage(SourcePollingChannelAdapter.java:250)
at org.springframework.integration.endpoint.AbstractPollingEndpoint.doPoll(AbstractPollingEndpoint.java:359)
at org.springframework.integration.endpoint.AbstractPollingEndpoint.pollForMessage(AbstractPollingEndpoint.java:328)
at org.springframework.integration.endpoint.AbstractPollingEndpoint.lambda$null$1(AbstractPollingEndpoint.java:275)
at org.springframework.integration.util.ErrorHandlingTaskExecutor.lambda$execute$0(ErrorHandlingTaskExecutor.java:57)
at org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:50)
at org.springframework.integration.util.ErrorHandlingTaskExecutor.execute(ErrorHandlingTaskExecutor.java:55)
at org.springframework.integration.endpoint.AbstractPollingEndpoint.lambda$createPoller$2(AbstractPollingEndpoint.java:272)
at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54)
at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:93)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run$$$capture(FutureTask.java:266)
at java.util.concurrent.FutureTask.run(FutureTask.java)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
The transformer is designed to always return a reply because it is a transformation operation. Therefore you can’t return null from your method . You get one-way error probably because your handle method is void.
We're calling SparkSQL job from Spark streaming. We're getting concurrent exception and Kafka consumer is closed error. Here is code and exception details:
Kafka consumer code
// Start reading messages from Kafka and get DStream
final JavaInputDStream<ConsumerRecord<String, byte[]>> consumerStream = KafkaUtils.createDirectStream(
getJavaStreamingContext(), LocationStrategies.PreferConsistent(),
ConsumerStrategies.<String, byte[]>Subscribe(SparkServiceConfParams.AIR.CONSUME_TOPICS,
sparkServiceConf.getKafkaConsumeParams()));
ThreadContext.put(Constants.CommonLiterals.LOGGER_UID_VAR, CommonUtils.loggerUniqueId());
// Decode each binary message and generate JSON array
JavaDStream<String> decodedStream = messagesStream.map(new Function<byte[], String>() {}
..
// publish generated json gzip to kafka
decodedStream.foreachRDD(new VoidFunction<JavaRDD<String>>() {
private static final long serialVersionUID = 1L;
#Override
public void call(JavaRDD<String> jsonRdd4DF) throws Exception {
//Dataset<Row> json = sparkSession.read().json(jsonRdd4DF);
if(!jsonRdd4DF.isEmpty()) {
//JavaRDD<String> jsonRddDF = getJavaSparkContext().parallelize(jsonRdd4DF.collect());
Dataset<Row> json = sparkSession.read().json(jsonRdd4DF);
SparkAIRMainJsonProcessor airMainJsonProcessor = new SparkAIRMainJsonProcessor();
AIRDataSetBean processAIRData = airMainJsonProcessor.processAIRData(json, sparkSession);
Error Details
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.util.ConcurrentModificationException: KafkaConsumer is not safe for multi-threaded access
Finally Kafka consumer closed:
org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand$$anonfun$run$1$$anonfun$apply$mcV$sp$1.apply(InsertIntoHadoopFsRelationCommand.scala:143)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:70)
at org.apache.spark.scheduler.Task.run(Task.scala:86)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.IllegalStateException:
This consumer has already been closed.
This issue is resolved using the Cache or Persist option of Spark streaming. In this scenario using cache RDD is not read from Kafka again and issue is resolved. It enables the concurrent usage of stream . But please use wisely cache option.Here is code:
JavaDStream<ConsumerRecord<String, byte[]>> cache = consumerStream.cache();
I want to get an appropriate exception message from Spark JDBC driver.
My test case:
Using Spark ver2.0.2 to access DB through JDBC Driver
At first locked DB, then try write to DB (write mode are append and overwrite)
Get exception message
Exception message I got:
case1 append: Cause already initialized <-Not good
case2 overwrite: DB2 SQL Error: SQLCODE=-913, SQLSTATE=57033, SQLERRMC=00C9008E;00000210;DSN00009.DRITEMRI.00000001, DRIVER=4.19.56 <-Good
My question:
How can I get an exception message like DB2 SQL Error: SQLCODE=-913, SQLSTATE=57033~ in 'case1 append'?
I guss a reason is function savePartitions (this function is called when Spark execute saveTable) doesn't show good exception. But I don't know how to fix it.
Here is detail exception message of Spark shell
case1 append:
scala> prodtbl.write.mode("Append").jdbc(url3,"DB2.D_ITEM_INFO",prop1)
17/04/03 17:50:26 ERROR Executor: Exception in task 0.0 in stage 1.0 (TID 1)
java.lang.IllegalStateException: Cause already initialized
at java.lang.Throwable.setCause(Throwable.java:365)
at java.lang.Throwable.initCause(Throwable.java:341)
at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.savePartition(JdbcUtils.scala:241)
at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$saveTable$1.apply(JdbcUtils.scala:300)
at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$saveTable$1.apply(JdbcUtils.scala:299)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$28.apply(RDD.scala:902)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$28.apply(RDD.scala:902)
at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1899)
at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1899)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:70)
at org.apache.spark.scheduler.Task.run(Task.scala:86)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1153)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.lang.Thread.run(Thread.java:785)
case2 overwrite:
scala> prodtbl.write.mode("Overwrite").jdbc(url3,"DB2.D_ITEM_INFO",prop1)
com.ibm.db2.jcc.am.SqlException: DB2 SQL Error: SQLCODE=-913, SQLSTATE=57033, SQLERRMC=00C9008E;00000210;DSN00009.DRITEMRI.00000001, DRIVER=4.19.56
at com.ibm.db2.jcc.am.kd.a(Unknown Source)
at com.ibm.db2.jcc.am.kd.a(Unknown Source)
at com.ibm.db2.jcc.am.kd.a(Unknown Source)
at com.ibm.db2.jcc.am.fp.c(Unknown Source)
at com.ibm.db2.jcc.am.fp.d(Unknown Source)
at com.ibm.db2.jcc.am.fp.b(Unknown Source)
at com.ibm.db2.jcc.t4.bb.i(Unknown Source)
at com.ibm.db2.jcc.t4.bb.c(Unknown Source)
at com.ibm.db2.jcc.t4.p.b(Unknown Source)
at com.ibm.db2.jcc.t4.vb.h(Unknown Source)
at com.ibm.db2.jcc.am.fp.jb(Unknown Source)
at com.ibm.db2.jcc.am.fp.a(Unknown Source)
at com.ibm.db2.jcc.am.fp.c(Unknown Source)
at com.ibm.db2.jcc.am.fp.executeUpdate(Unknown Source)
at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.dropTable(JdbcUtils.scala:94)
at org.apache.spark.sql.DataFrameWriter.jdbc(DataFrameWriter.scala:422)
... 48 elided
So initCause described as below:
public Throwable initCause(Throwable cause)
Initializes the cause of this throwable to the specified value. (The cause is the throwable that caused this throwable to get thrown.)
This method can be called at most once. It is generally called from within the constructor, or immediately after creating the throwable. If this throwable was created with Throwable(Throwable) or Throwable(String,Throwable), this method cannot be called even once.
An example of using this method on a legacy throwable type without other support for setting the cause is:
try {
lowLevelOp();
} catch (LowLevelException le) {
throw (HighLevelException)
new HighLevelException().initCause(le); // Legacy constructor
}
Parameters:
cause - the cause (which is saved for later retrieval by the getCause() method). (A null value is permitted, and indicates that the cause is nonexistent or unknown.)
Returns:
a reference to this Throwable instance.
Throws:
IllegalArgumentException - if cause is this throwable. (A throwable cannot be its own cause.)
IllegalStateException - if this throwable was created with Throwable(Throwable) or Throwable(String,Throwable), or this method has already been called on this throwable.
And it means the cause was something not originated to initialize with initCase method and you haven't received what you waited for.
My application is about parsing and processing a huge file and that by invoking dao methods to check existing of some attributs in a sybase database.
We have used for that spring batch and multi-threading mechanism and so that we have occured an exception in afterRead method of the class ItemReadListener caused by an SQLException: Invalid state, the Connection object is closed."
org.springframework.jdbc.UncategorizedSQLException: PreparedStatementCallback; uncategorized SQLException for SQL [select count(*) from USER_SERVICE where user_serv_oidval = ?]; SQL state [HY010]; error code [0]; Invalid state, the Connection object is closed.; nested exception is java.sql.SQLException: Invalid state, the Connection object is closed.
at org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:84)
at org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:81)
at org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:81)
at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:660)
at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:695)
at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:727)
at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:737)
at org.springframework.jdbc.core.JdbcTemplate.queryForObject(JdbcTemplate.java:805)
at org.springframework.jdbc.core.JdbcTemplate.queryForObject(JdbcTemplate.java:824)
at com.orange.essentials.dao.impl.ServiceDaoImpl.isExistServiceById(ServiceDaoImpl.java:28)
at com.orange.essentials.services.ItemReadListenerFunctionalValidator.afterRead(ItemReadListenerFunctionalValidator.java:51)
at com.orange.essentials.services.ItemReadListenerFunctionalValidator.afterRead(ItemReadListenerFunctionalValidator.java:1)
at org.springframework.batch.core.listener.CompositeItemReadListener.afterRead(CompositeItemReadListener.java:60)
at org.springframework.batch.core.listener.MulticasterBatchListener.afterRead(MulticasterBatchListener.java:232)
at org.springframework.batch.core.step.item.SimpleChunkProvider.doRead(SimpleChunkProvider.java:93)
at org.springframework.batch.core.step.item.SimpleChunkProvider.read(SimpleChunkProvider.java:155)
at org.springframework.batch.core.step.item.SimpleChunkProvider$1.doInIteration(SimpleChunkProvider.java:114)
at org.springframework.batch.repeat.support.RepeatTemplate.getNextResult(RepeatTemplate.java:368)
at org.springframework.batch.repeat.support.RepeatTemplate.executeInternal(RepeatTemplate.java:215)
at org.springframework.batch.repeat.support.RepeatTemplate.iterate(RepeatTemplate.java:144)
at org.springframework.batch.core.step.item.SimpleChunkProvider.provide(SimpleChunkProvider.java:108)
at org.springframework.batch.core.step.item.ChunkOrientedTasklet.execute(ChunkOrientedTasklet.java:69)
at org.springframework.batch.core.step.tasklet.TaskletStep$ChunkTransactionCallback.doInTransaction(TaskletStep.java:406)
at org.springframework.batch.core.step.tasklet.TaskletStep$ChunkTransactionCallback.doInTransaction(TaskletStep.java:330)
at org.springframework.transaction.support.TransactionTemplate.execute(TransactionTemplate.java:133)
at org.springframework.batch.core.step.tasklet.TaskletStep$2.doInChunkContext(TaskletStep.java:271)
at org.springframework.batch.core.scope.context.StepContextRepeatCallback.doInIteration(StepContextRepeatCallback.java:77)
at org.springframework.batch.repeat.support.TaskExecutorRepeatTemplate$ExecutingRunnable.run(TaskExecutorRepeatTemplate.java:262)
at java.lang.Thread.run(Thread.java:744)
Caused by: java.sql.SQLException: Invalid state, the Connection object is closed.
at net.sourceforge.jtds.jdbc.ConnectionJDBC2.checkOpen(ConnectionJDBC2.java:1714)
at net.sourceforge.jtds.jdbc.ConnectionJDBC2.prepareStatement(ConnectionJDBC2.java:2435)
at com.mchange.v2.c3p0.impl.NewProxyConnection.prepareStatement(NewProxyConnection.java:213)
at org.springframework.jdbc.core.JdbcTemplate$SimplePreparedStatementCreator.createPreparedStatement(JdbcTemplate.java:1557)
at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:638)