Dispatcher has no subscribers error for ftp - spring-integration

I have a file dropped at the ftp location which should be picked up by ftp-inbound-adapter. This file is saved to a local-directory. This local-directory is in turn polled by spring file-inbound-adapter. The filenamegenerator bean is used in the file-inbound-adapter and decides the destination dynamically. I have also posted another question about the file in the local-directory not being deleted. This is the problem I am facing.
This is a my entire configuration
<util:properties id="someid" location="classpath:config/config.properties"/>
<mvc:annotation-driven />
<context:component-scan base-package="com.dms" />
<bean
class="org.springframework.web.servlet.view.InternalResourceViewResolver">
<property name="prefix">
<value>/WEB-INF/</value>
</property>
<property name="suffix">
<value>.jsp</value>
</property>
</bean>
<context:property-placeholder location="classpath:config/jdbc.properties,classpath:config/config.properties,classpath:config/ftp.properties"/>
<bean id="dataSource"
class="org.springframework.jdbc.datasource.DriverManagerDataSource"
>
<property name="driverClassName" value="${jdbc.driverClassName}"/>
<property name="url" value="${jdbc.url}"/>
<property name="username" value="${jdbc.username}"/>
<property name="password" value="${jdbc.password}"/>
</bean>
<bean id="sessionFactory"
class="org.springframework.orm.hibernate4.LocalSessionFactoryBean">
<property name="dataSource">
<ref bean="dataSource" />
</property>
<property name="hibernateProperties">
<props>
<prop key="hibernate.dialect">${hibernate.dialect}</prop>
<prop key="hibernate.show_sql">${hibernate.show_sql}</prop>
<prop key="hibernate.format_sql">${hibernate.format_sql}</prop>
<prop key="hibernate.generate_statistics">${hibernate.generate_statistics}</prop>
</props>
</property>
<property name="packagesToScan">
<list>
<value>com.dms.entity</value>
</list>
</property>
</bean>
<tx:annotation-driven />
<bean id="transactionManager"
class="org.springframework.orm.hibernate4.HibernateTransactionManager">
<property name="sessionFactory" ref="sessionFactory" />
</bean>
<bean id="multipartResolver"
class="org.springframework.web.multipart.commons.CommonsMultipartResolver">
<!-- setting maximum upload size -->
<property name="maxUploadSize" value="10485760" />
</bean>
<!-- scheduler to pickup temp folder files to permanent location -->
<bean class="org.springframework.scheduling.quartz.SchedulerFactoryBean">
<property name="triggers">
<list>
<ref bean="simpleTrigger" />
</list>
</property>
</bean>
<bean id="dmsFilesDetectionJob" class="com.dms.scheduler.job.DMSFilesDetectionJob">
</bean>
<bean id="dmsFilesDetectionJobDetail" class="org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean">
<property name="targetObject" ref="dmsFilesDetectionJob" />
<property name="targetMethod" value="pollTempFolder" />
<property name="concurrent" value="false" />
</bean>
<bean id="simpleTrigger" class="org.springframework.scheduling.quartz.CronTriggerBean">
<property name="jobDetail" ref="dmsFilesDetectionJobDetail" />
<!-- <property name="cronExpression" value="1 * * * * ?" /> -->
<property name="cronExpression" value="0 0/1 * * * ?" />
</bean>
<bean id="fileNameGenerator" class="com.dms.util.FileNameGenerator"/>
<int-file:inbound-channel-adapter id="filesIn" directory="file:${paths.root}" channel="abc" filter="compositeFilter" >
<int:poller id="poller" fixed-delay="5000" />
</int-file:inbound-channel-adapter>
<int:channel id="abc"/>
<bean id="compositeFilter" class="org.springframework.integration.file.filters.CompositeFileListFilter">
<constructor-arg>
<list>
<!-- Ensures that the file is whole before processing it -->
<bean class="com.dms.util.CustomFileFilter"/>
<!-- Ensures files are picked up only once from the directory -->
<bean class="org.springframework.integration.file.filters.AcceptOnceFileListFilter" />
</list>
</constructor-arg>
</bean>
<int-file:outbound-channel-adapter channel="abc" id="filesOut"
directory-expression="#outPathBean.getPath()"
delete-source-files="true" filename-generator="fileNameGenerator" />
<bean class="org.springframework.web.servlet.mvc.annotation.AnnotationMethodHandlerAdapter">
<property name="messageConverters">
<list>
<ref bean="jsonMessageConverter"/>
</list>
</property>
</bean>
<bean id="jsonMessageConverter" class="org.springframework.http.converter.json.MappingJackson2HttpMessageConverter">
<!-- <property name="prefixJson" value="false"/> -->
<!-- <property name="objectMapper">
<bean class="com.dms.util.HibernateAwareObjectMapper" />
</property> -->
<property name="supportedMediaTypes" value="application/json"/>
</bean>
<bean id="ftpClientFactory"
class="org.springframework.integration.ftp.session.DefaultFtpSessionFactory">
<property name="host" value="${ftp.ip}"/>
<property name="port" value="${ftp.port}"/>
<property name="username" value="${ftp.username}"/>
<property name="password" value="${ftp.password}"/>
<property name="clientMode" value="0"/>
<property name="fileType" value="2"/>
<property name="bufferSize" value="100000"/>
</bean>
<int-ftp:outbound-channel-adapter id="ftpOutbound"
channel="ftpChannel"
session-factory="ftpClientFactory"
charset="UTF-8"
remote-file-separator="/"
auto-create-directory="true"
remote-directory="."
use-temporary-file-name="true"
auto-startup="true"
/>
<int-ftp:inbound-channel-adapter id="ftpInbound"
channel="ftpChannel"
session-factory="ftpClientFactory"
charset="UTF-8"
local-directory="file:${paths.root}"
delete-remote-files="true"
temporary-file-suffix=".writing"
remote-directory="."
filename-pattern="${file.char}*${file.char}"
preserve-timestamp="true"
auto-startup="true">
<int:poller fixed-rate="1000"/>
</int-ftp:inbound-channel-adapter>
<int:channel id="ftpChannel" />
This is the error I am getting
18:02:34.655 E|LoggingHandler |org.springframework.messaging.MessageDeliveryException: Dispatcher has no subscribers for channel 'org.springframework.web.context.WebApplicationContext:/DMS/DMS-dispatcher.ftpChannel'.
This exception does not appear everytime.
As you can see I have added auto-startup="true". Have used unique id's for both the channels as well as adapters. Please let me know what is wrong here!
Thanks

I just had to deal with this but with a file inbound-channel-adapter. The issue is intermittent, and only at startup. I think adapters with pollers can start pulling in messages before Spring Integration has fully initialized.
My fix is to disable the adapter at startup. The details of the adapter are not so important, other than it has an id and is set to not autostart:
<!--
Read files from an "inbox" directory, placing them on an "inbox" channel...
-->
<int-file:inbound-channel-adapter id="inboxScanner"
directory="$import{inbox}"
auto-create-directory="true"
channel="fileInbox"
prevent-duplicates="false"
auto-startup="false">
<int:poller fixed-rate="$import{inbox.scan.rate.seconds}"
time-unit="SECONDS"
max-messages-per-poll="$import{inbox.max.imports.per.scan}"/>
</int-file:inbound-channel-adapter>
I then tap into Spring's application lifecycle events, and once the application context is finished being created (or refreshed), I tell the adapter to start:
<!--
Only start the scanner after the application has finished initializing...
-->
<int-event:inbound-channel-adapter event-types="org.springframework.context.event.ContextRefreshedEvent"
channel="contextRefreshEvents"/>
<int:publish-subscribe-channel id="contextRefreshEvents"/>
<int:outbound-channel-adapter channel="contextRefreshEvents"
expression="#inboxScanner.start()" />
The "event" components are from spring-integration-event.

First of all your integration flow ins't clear. But I see that your issue is for this
<int-ftp:inbound-channel-adapter id="ftpInbound" channel="ftpChannel"
Where there is really no one who is subscribed to that ftpChannel SubscribableChannel.
So, that adapter starts his work and sends message to that channel, but ... Dispatcher has no subscribers.
Try to fix that issue and figure out how to go ahead.
EDIT
Not sure what you have found in my answer so bad that it forced you to downvote it. Anyway.
There was some phase problem before, but as you see it has been fixed in the version 4.1.
So, to reach the immediate fix right now you should do this:
phase="0x7fffffff" // Integer.MAX_VALUE
by default phase is 0, so the inbound channel adapter may start before outbound channel adapter.
Or just upgrade to the latest Spring Integration!

Related

Azure Redis connection failure when using SSL

I am using Spring Session with Redis using Azure Redis.
Things are working fine with the non-SSL port 6379. However with the SSL port 6380, I get this error:
ERROR (org.springframework.data.redis.listener.RedisMessageListenerContainer:651) || - Connection failure occurred. Restarting subscription task after 5000 ms
That’s it. No further information.
Here is my Redis configuration:
<bean id="redisPassword" class="org.springframework.data.redis.connection.RedisPassword">
<constructor-arg index="0" value="${spring.redis.password}"/>
</bean>
<bean id="redisStandaloneConfiguration"
class="org.springframework.data.redis.connection.RedisStandaloneConfiguration">
<property name="hostName" value="${spring.redis.host}"/>
<property name="port" value="${spring.redis.port}"/>
<property name="password" ref="redisPassword"/>
</bean>
<util:constant id="configureRedisAction"
static-field="org.springframework.session.data.redis.config.ConfigureRedisAction.NO_OP"/>
<bean id="lettuceClientConfiguration"
class="org.springframework.data.redis.connection.lettuce.DefaultLettuceClientConfiguration"
factory-method="defaultConfiguration">
</bean>
<context:annotation-config/>
<bean class="org.springframework.session.data.redis.config.annotation.web.http.RedisHttpSessionConfiguration"
p:configureRedisAction-ref="configureRedisAction"/>
<bean class="org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory">
<constructor-arg index="0" ref="redisStandaloneConfiguration"/>
<constructor-arg index="1" ref="lettuceClientConfiguration"/>
</bean>
What is causing the connection failure?
<util:constant id="configureRedisAction"
static-field="org.springframework.session.data.redis.config.ConfigureRedisAction.NO_OP"/>
<context:annotation-config/>
<bean class="org.springframework.session.data.redis.config.annotation.web.http.RedisHttpSessionConfiguration"
p:configureRedisAction-ref="configureRedisAction"/>
<bean id="jedisPoolConfig" class="redis.clients.jedis.JedisPoolConfig">
<property name="maxTotal" value="200" />
<property name="maxIdle" value="50" />
<property name="maxWaitMillis" value="30000" />
<property name="minIdle" value="10"/>
</bean>
<bean class="org.springframework.data.redis.connection.jedis.JedisConnectionFactory">
<property name="hostName" value="${spring.redis.host}" />
<property name="port" value="${spring.redis.port}" />
<property name="poolConfig" ref="jedisPoolConfig" />
<property name="usePool" value="true" />
<property name="useSsl" value="${spring.redis.ssl}"/>
<property name="password" value="${spring.redis.password}"/>
</bean>

dynamic delay configuration in spring poller

Polling dynamically
I am using poller component from integration to poll files from s3.The fixed delay is 15 min and max message rate is 1 .The reason i did this was in down stream messages in xd were clogging since I am using http.Now this is file for 100k records file but when file size is small i still wait 15 min though i can process fast.Now is there any way to dynamically set delay depending in size of file.Because we don't know which files will be polled also to know it?Depending on file size i will pick up or number of records can we change dynamically the fixed delay or fixed rate?
<int:poller fixed-delay="${fixedDelay}" default="true" max-messages-per-poll="${maxMessageRate}">
<int:advice-chain>
<ref bean="pollAdvise"/>
</int:advice-chain>
</int:poller>
<bean id="pollAdvise" class="org.springframework.integration.scheduling.PollSkipAdvice">
<constructor-arg ref="healthCheckStrategy"/>
</bean>
<bean id="healthCheckStrategy" class="test.ServiceHealthCheckPollSkipStrategy">
<property name="url" value="${url}"/>
<property name="doHealthCheck" value="${doHealthCheck}"/>
</bean>
<bean id="credentials" class="org.springframework.integration.aws.core.BasicAWSCredentials">
<property name="accessKey" value="${accessKey}"/>
<property name="secretKey" value="${secretKey}"/>
</bean>
<bean id="clientConfiguration" class="com.amazonaws.ClientConfiguration">
<property name="proxyHost" value="${proxyHost}"/>
<property name="proxyPort" value="${proxyPort}"/>
<property name="preemptiveBasicProxyAuth" value="false"/>
</bean>
<bean id="s3Operations" class="org.springframework.integration.aws.s3.core.CustomC1AmazonS3Operations">
<constructor-arg index="0" ref="credentials"/>
<constructor-arg index="1" ref="clientConfiguration"/>
<property name="awsEndpoint" value="s3.amazonaws.com"/>
<property name="temporaryDirectory" value="${temporaryDirectory}"/>
<property name="awsSecurityKey" value="${awsSecurityKey}"/>
</bean>
<!-- aws-endpoint="https://s3.amazonaws.com" -->
<int-aws:s3-inbound-channel-adapter aws-endpoint="s3.amazonaws.com"
bucket="${bucket}"
s3-operations="s3Operations"
credentials-ref="credentials"
file-name-wildcard="${fileNameWildcard}"
remote-directory="${remoteDirectory}"
channel="splitChannel"
local-directory="${localDirectory}"
accept-sub-folders="false"
delete-source-files="true"
archive-bucket="${archiveBucket}"
archive-directory="${archiveDirectory}">
</int-aws:s3-inbound-channel-adapter>
<int-file:splitter id="s3splitter" input-channel="splitChannel" output-channel="bridge" markers="false" charset="UTF-8">
<int-file:request-handler-advice-chain>
<bean class="org.springframework.integration.handler.advice.ExpressionEvaluatingRequestHandlerAdvice">
<property name="onSuccessExpression" value="payload.delete()"/>
</bean>
</int-file:request-handler-advice-chain>
</int-file:splitter>
Starting with Spring Integration 4.2 the AbstractMessageSourceAdvice has been introduced:
This method is called after the receive() method; again, you can reconfigure the source, or take any action perhaps depending on the result (which can be null if there was no message created by the source). You can even return a different message!
Starting with version 4.3 we introduce CompoundTriggerAdvice: http://docs.spring.io/spring-integration/docs/4.3.0.BUILD-SNAPSHOT/reference/html/messaging-channels-section.html#_compoundtriggeradvice
Which you can use for your use-case based on the payload size.

Spring Integration auto reconnect to IBM MQ thru JBOSS Resource Adapter

We are using Spring Integration in our project and we have a requirement where If IBM MQ goes down then we will have to auto connect to IBM MQ when it is up. We have done this implementation using recoveryInterval option of org.springframework.jms.listener.DefaultMessageListenerContainer class. We have given recovery interval value to try to recover the MQ connection. But it is not recovering the connection after MQ restart. Below was my existing configuration:
<jms:message-driven-channel-adapter id="adapterId" channel="raw-channel" container="messageListenerContainer" />
<bean id="messageListenerContainer" class="org.springframework.jms.listener.DefaultMessageListenerContainer">
<property name="connectionFactory" ref="customQueueCachingConnectionFactory" />
<property name="destination" ref="requestQueue" />
<property name="recoveryInterval" value="60000" />
</bean>
Below is the Current Connection Factory :
<bean id="queueCachingConnectionFactory"
class="org.springframework.jms.connection.CachingConnectionFactory">
<property name="targetConnectionFactory" ref="queueConnectionFactory" />
<property name="sessionCacheSize" value="10" />
<property name="cacheProducers" value="false" />
<!-- <property name="reconnectOnException" value="true" /> -->
<!-- <property name="exceptionListener" ref="MQExceptionListener"></property> -->
</bean>
<jee:jndi-lookup id="queueConnectionFactory" jndi-name="MQConnectionFactory"
expected-type="javax.jms.ConnectionFactory" lookup-on-startup="true"></jee:jndi-lookup>
<jee:jndi-lookup id="queue" jndi-name="Queue"
expected-type="javax.jms.Queue" lookup-on-startup="true"/>
ERROR [task-scheduler-4] LoggingHandler:145 -org.springframework.jms.IllegalStateException: MQJCA1019: The connection is closed.; nested exception is com.ibm.msg.client.jms.DetailedIllegalStateException: MQJCA1019: The connection is closed.
The application attempted to use a JMS connection after it had closed the connection.
Modify the application so that it closes the JMS connection only after it has finished using the connection.
Thanks in Advance!!
The default message listening container should reference the caching connection factory:
<!-- caching connection factory fascade, also implements exception listener -->
<bean id="cachingConnectionFactory" class="org.springframework.jms.connection.CachingConnectionFactory">
<property name="targetConnectionFactory" ref="connectionFactory"/>
<property name="sessionCacheSize" value="10"/>
<property name="reconnectOnException" value="true"/>
</bean>
<!-- this is the Message Driven POJO (MDP) -->
<bean id="messageDrivenPOJO" class="com.redhat.gss.spring.MessageDrivenPOJO" />
<!-- The message listener container -->
<bean id="messageListener" class="org.springframework.jms.listener.DefaultMessageListenerContainer">
<property name="sessionTransacted" value="true"/>
<property name="concurrentConsumers" value="1"/>
<property name="cacheLevelName" value="CACHE_CONSUMER"/>
<property name="receiveTimeout" value="10000"/>
<property name="sessionAcknowledgeMode" value="2"/>
<property name="messageListener" ref="messageDrivenPOJO"/>
<property name="connectionFactory" ref="cachingConnectionFactory"/>
<property name="exceptionListener" ref="cachingConnectionFactory"/>
<property name="destination" ref="jbossQueue"/>
</bean>

how to set a reconnect value with CachingConnectionFactory

In my app the message comes to the inque and then is sent to the output queue. We do this thru spring integration. My requirement is if there is a problem in connecting to the output queue then it shud try reconnecting 3 times with a delay of 30 secs and finally if it fails then log the exception. Can you please help on how to achieve this ? My config file -
<bean id="mqQcfParent" class="com.ibm.mq.jms.MQQueueConnectionFactory">
<property name="transportType">
<util:constant static-field="com.ibm.mq.jms.JMSC.MQJMS_TP_CLIENT_MQ_TCPIP"/>
</property>
<property name="hostName" value="${mq.out.hostname}"/>
<property name="channel" value="${mq.out.channel}"/>
<property name="port" value="${mq.out.port}"/>
</bean>
<bean id="remoteConnectionFactory" class="org.springframework.jms.connection.CachingConnectionFactory">
<property name="targetConnectionFactory" ref="mqQcfParent"/>
<property name="sessionCacheSize" value="${mq.out.cacheSize}"/>
<property name="cacheProducers" value="true"/>
<property name="cacheConsumers" value="true"/>
</bean>
<bean id="inQueue" class="com.ibm.mq.jms.MQQueue">
<property name="baseQueueName" value="${mq.in.queue}"/>
</bean>
<bean id="aircraftAssignQueue" class="com.ibm.mq.jms.MQQueue">
<property name="baseQueueName" value="${mq.out.aircraftAssignQueue}"/>
</bean>
</bean>
<bean id="failureQueue" class="com.ibm.mq.jms.MQQueue">
<property name="baseQueueName" value="${mq.out.failureQueue}"/>
</bean>
<bean id="messageListenerContainerParent" class="org.springframework.jms.listener.DefaultMessageListenerContainer" abstract="true">
<property name="destination" ref="inQueue"/>
<property name="sessionTransacted" value="true"/>
<property name="maxConcurrentConsumers" value="${mq.in.max.consumer}"/>
<property name="concurrentConsumers" value="${mq.in.min.consumer}"/>
<property name="receiveTimeout" value="5000"/>
<property name="recoveryInterval" value="60000"/>
<property name="autoStartup" value="true"/>
</bean>
<bean id="messageListenerContainerCDC" parent="messageListenerContainerParent">
<property name="connectionFactory">
<bean parent="remoteConnectionFactory">
<property name="targetConnectionFactory">
<bean parent="mqQcfParent">
<property name="hostName" value="${mq.in.cdc.hostname}"/>
<property name="channel" value="${mq.in.cdc.channel}"/>
<property name="port" value="${mq.in.cdc.port}"/>
</bean>
</property>
</bean>
</property>
</bean>
<bean id="messageListenerContainerPDC" parent="messageListenerContainerParent">
<property name="connectionFactory">
<bean parent="remoteConnectionFactory">
<property name="targetConnectionFactory">
<bean parent="mqQcfParent">
<property name="hostName" value="${mq.in.pdc.hostname}"/>
<property name="channel" value="${mq.in.pdc.channel}"/>
<property name="port" value="${mq.in.pdc.port}"/>
</bean>
</property>
</bean>
</property>
</bean>
</beans>
<context:property-placeholder location="config/application.properties"/>
<import resource="jms-listener-container-config.xml"/>
<!-- Get Input Messages -->
<int-jms:message-driven-channel-adapter id="msgInCDC" channel="toRoute" container="messageListenerContainerCDC" error-channel="errorChannel" acknowledge="transacted"/>
<int-jms:message-driven-channel-adapter id="msgInPDC" channel="toRoute" container="messageListenerContainerPDC" error-channel="errorChannel" acknowledge="transacted"/>
<!-- Route Messages Depending on Root Element -->
<int:channel id="toAircraftAssign"/>
<int:channel id="toDiversionChange"/>
<int:channel id="toFlightCreate"/>
<int:channel id="toFlightPlanRelease"/>
<int:channel id="toGateChange"/>
<int:channel id="toPositionReport"/>
<int:channel id="toScheduleChange"/>
<int-xml:xpath-router id="flightUpdateRouter" input-channel="toRoute" default-output-channel="errorChannel" evaluate-as-string="true">
<int-xml:xpath-expression expression="name(/*)"/>
<int-xml:mapping value="AircraftAssignment" channel="toAircraftAssign"/>
<int-xml:mapping value="DiversionChangesUpdate" channel="toDiversionChange"/>
<int-xml:mapping value="CreateFlight" channel="toFlightCreate"/>
<int-xml:mapping value="FlightPlanRelease" channel="toFlightPlanRelease"/>
<int-xml:mapping value="GateChange" channel="toGateChange"/>
<int-xml:mapping value="PositionReportUpdate" channel="toPositionReport"/>
<int-xml:mapping value="ScheduleChangesUpdate" channel="toScheduleChange"/>
</int-xml:xpath-router>
<int-jms:outbound-channel-adapter id="aircraftAssignMsgOut" channel="toAircraftAssign" connection-factory="remoteConnectionFactory" destination="aircraftAssignQueue"/>
<int-jms:outbound-channel-adapter id="diversionChangeMsgOut" channel="toDiversionChange" connection-factory="remoteConnectionFactory" destination="diversionChangeQueue"/>
<int-jms:outbound-channel-adapter id="flightCreateMsgOut" channel="toFlightCreate" connection-factory="remoteConnectionFactory" destination="flightCreateQueue"/>
<int-jms:outbound-channel-adapter id="flightPlanReleaseMsgOut" channel="toFlightPlanRelease" connection-factory="remoteConnectionFactory" destination="flightPlanReleaseQueue"/>
<int-jms:outbound-channel-adapter id="gateChangeMsgOut" channel="toGateChange" connection-factory="remoteConnectionFactory" destination="gateChangeQueue"/>
<int-jms:outbound-channel-adapter id="positionReportMsgOut" channel="toPositionReport" connection-factory="remoteConnectionFactory" destination="positionReportQueue"/>
<int-jms:outbound-channel-adapter id="scheduleChangeMsgOut" channel="toScheduleChange" connection-factory="remoteConnectionFactory" destination="scheduleChangeQueue"/>
<!-- Error Handling -->
<int-jms:outbound-channel-adapter id="errMsgOut" channel="errorChannel" connection-factory="remoteConnectionFactory" destination="failureQueue"/>
<!-- Logger -->
<int:wire-tap pattern="to*" order="7" channel="wireTapChannel"/>
<int:logging-channel-adapter id="wireTapChannel" level="debug" logger-name="WIRETAP"/>
<!-- Logger -->
<!--
<int:wire-tap pattern="to*" order="0" channel="loggerChannel"/>
-->
<!--
<int:logging-channel-adapter id="loggerChannel" level="DEBUG" expression="'YYYY'"/>
-->
</beans>
You don't show any Spring Integration configuration but, presuming you are using a JMS outbound channel adapter, you an add a retry advice with an appropriately configured SimplyRetryPolicy.
However, if the same broker is being used for the inbound queue too, that session will be broken and the message redelivered anyway; so you might be better off setting the retry policy in the broker.

ftp outbound channel adapter not transferring same files

I have the following configuration in my application. I am writing a file to the ftp folder and reading from the ftp location.
I want to take the file from the ftp location and save it in a directory decided dynamically. This is done by using two chained channel adapters. The ftp nbound channel adapter picks up the file, puts it in the local directory and then the file inbound channel adapter picks up the file and puts it in its final destination. I need to filter old files for this process. ftp inbound channel adapter custom filter gave me FTPFile object in the filtering method. This object gives the last modified date rather than the date the file was put in the filter. Due to this limitation I had to use file inbound channel adapter as well.
Since I do not know how to generate the source directory dynamically, I am using plain java code to copy the required file to the local-directory from where the ftp outbound channel picks it up and puts it on ftp location.
This is the configuration:
<bean id="fileNameGenerator" class="com.polling.util.FileNameGenerator"/>
<int-file:inbound-channel-adapter id="filesIn" directory="file:${paths.root}" channel="abc" filter="compositeFilter" >
<int:poller id="poller" fixed-rate="500" />
</int-file:inbound-channel-adapter>
<int:channel id="abc"/>
<bean id="compositeFilter" class="org.springframework.integration.file.filters.CompositeFileListFilter">
<constructor-arg>
<list>
<!-- Ensures that the file is whole before processing it -->
<bean class="com.polling.util.CustomFileFilter"/>
<!-- Ensures files are picked up only once from the directory -->
<bean class="org.springframework.integration.file.filters.AcceptOnceFileListFilter" />
</list>
</constructor-arg>
</bean>
<int-file:outbound-channel-adapter channel="abc" id="filesOut"
directory-expression="#outPathBean.getPath()"
delete-source-files="true" filename-generator="fileNameGenerator" >
<int-file:request-handler-advice-chain>
<bean class="org.springframework.integration.handler.advice.ExpressionEvaluatingRequestHandlerAdvice">
<property name="onSuccessExpression" value="payload.delete()" />
</bean>
</int-file:request-handler-advice-chain>
</int-file:outbound-channel-adapter>
<bean id="ftpClientFactory"
class="org.springframework.integration.ftp.session.DefaultFtpSessionFactory">
<property name="host" value="${ftp.ip}"/>
<property name="port" value="${ftp.port}"/>
<property name="username" value="${ftp.username}"/>
<property name="password" value="${ftp.password}"/>
<property name="clientMode" value="0"/>
<property name="fileType" value="2"/>
<property name="bufferSize" value="100000"/>
</bean>
<int:channel id="ftpChannel"/>
<int-ftp:inbound-channel-adapter id="ftpInbound"
channel="ftpChannel"
session-factory="ftpClientFactory"
charset="UTF-8"
local-directory="file:${paths.root}"
delete-remote-files="true"
temporary-file-suffix=".writing"
remote-directory="${file.ftpfolder}"
filter="compositeFilterRemote"
preserve-timestamp="true"
auto-startup="true">
<int:poller fixed-rate="1000"/>
</int-ftp:inbound-channel-adapter>
<int-ftp:outbound-channel-adapter id="ftpOutbound"
channel="ftpChannel"
session-factory="ftpClientFactory"
charset="UTF-8"
remote-file-separator="/"
auto-create-directory="true"
remote-directory="${file.ftpfolder}"
use-temporary-file-name="true"
temporary-file-suffix=".writing">
<int-ftp:request-handler-advice-chain>
<bean class="org.springframework.integration.handler.advice.ExpressionEvaluatingRequestHandlerAdvice">
<property name="onSuccessExpression" value="payload.delete()" />
</bean>
</int-ftp:request-handler-advice-chain>
</int-ftp:outbound-channel-adapter>
Using the suggestion in #Gary's answer, I have added a CustomFilter to the ftp inbound channel filter attribute. This checks the regular expression of the file name as follows
#Override
public List<FTPFile> filterFiles(FTPFile[] files)
{
List<FTPFile> ret = new ArrayList<FTPFile>();
Pattern pattern = Pattern.compile("~.*?~");
Matcher matcher;
for (FTPFile file : files)
{
matcher = pattern.matcher(file.getName());
if(matcher.matches())
{
ret.add(file);
}
}
return ret;
}
Due to this the ~something~ files are picked up and sent to final destination.
My Questions are:
The files being sent from the final destination to the ftp location have a different pattern which I don't know where to put(something#something#something).
Sometimes the ftp folder may be tampered with and the file deleted. If so I would like spring to rewrite the file. The same file gets written again in the local-directory mentioned in the <ftp:inbound-channel-adapter>. But ftp outbound channel adapter does not pick the file up and put it in the ftp location. According to #Gary's answer I need to configure the filter for the inbound channel adapter to avoid the accept once filter.
Does this mean that I should create two separate channels and two different flows one for back and one for forth? Is this the right way forward for my requirement?
Thanks for your help
EDIT::
I have tried to implement two different processes. One to pickup files from remotedir with pattern ~something~ and another from the local directory with pattern something#something#something. Though the basic functionality is working, if the file in the remote dir is deleted i cannot execute the second flow again. As Gary said the accept all filter needs to be in place but I dont know how to use both the custom filter and the accept all filter.
Any suggestions are appreciated
The AcceptOnceFileListFilter is configured by default. Use the local-filter on the inbound adapter to use a different filter (such as an AcceptAllFileListFilter). See the documentation.
That said, your application looks odd - it appears you are fetching files and simply sending them back.
This is what worked
I continued with the two flows and changed my bean configuration to include the accept all filter.
This is the configuration now.
<bean id="fileNameGenerator" class="com.polling.util.FileNameGenerator"/>
<int-file:inbound-channel-adapter id="filesIn" directory="file:${paths.root}" channel="abc" filter="compositeFilter" >
<int:poller id="poller" fixed-rate="500" />
</int-file:inbound-channel-adapter>
<int:channel id="abc"/>
<bean id="compositeFilter" class="org.springframework.integration.file.filters.CompositeFileListFilter">
<constructor-arg>
<list>
<!-- Ensures that the file is whole before processing it -->
<bean class="com.polling.util.CustomFileFilter"/>
<!-- Ensures files are picked up only once from the directory -->
<bean class="org.springframework.integration.file.filters.AcceptOnceFileListFilter" />
</list>
</constructor-arg>
</bean>
<int-file:outbound-channel-adapter channel="abc" id="filesOut"
directory-expression="#outPathBean.getPath()"
delete-source-files="true" filename-generator="fileNameGenerator" >
<int-file:request-handler-advice-chain>
<bean class="org.springframework.integration.handler.advice.ExpressionEvaluatingRequestHandlerAdvice">
<property name="onSuccessExpression" value="payload.delete()" />
</bean>
</int-file:request-handler-advice-chain>
</int-file:outbound-channel-adapter>
<bean id="ftpClientFactory"
class="org.springframework.integration.ftp.session.DefaultFtpSessionFactory">
<property name="host" value="${ftp.ip}"/>
<property name="port" value="${ftp.port}"/>
<property name="username" value="${ftp.username}"/>
<property name="password" value="${ftp.password}"/>
<property name="clientMode" value="0"/>
<property name="fileType" value="2"/>
<property name="bufferSize" value="100000"/>
</bean>
<int:channel id="ftpChannel1"/>
<int-ftp:inbound-channel-adapter id="ftpInbound1"
channel="ftpChannel1"
session-factory="ftpClientFactory"
charset="UTF-8"
local-directory="file:${paths.root}"
delete-remote-files="true"
temporary-file-suffix=".writing"
remote-directory="."
preserve-timestamp="true"
auto-startup="true"
local-filter="compositeFilterLocal">
<int:poller fixed-rate="1000"/>
</int-ftp:inbound-channel-adapter>
<int-ftp:outbound-channel-adapter id="ftpOutbound1"
channel="ftpChannel1"
session-factory="ftpClientFactory"
charset="UTF-8"
remote-file-separator="/"
auto-create-directory="true"
remote-directory="${file.ftpfolder}"
use-temporary-file-name="true"
temporary-file-suffix=".writing">
<int-ftp:request-handler-advice-chain>
<bean class="org.springframework.integration.handler.advice.ExpressionEvaluatingRequestHandlerAdvice">
<property name="onSuccessExpression" value="payload.delete()" />
</bean>
</int-ftp:request-handler-advice-chain>
</int-ftp:outbound-channel-adapter>
<int:channel id="ftpChannel"/>
<int-ftp:inbound-channel-adapter id="ftpInbound"
channel="ftpChannel"
session-factory="ftpClientFactory"
charset="UTF-8"
local-directory="file:${paths.root}"
delete-remote-files="true"
temporary-file-suffix=".writing"
remote-directory="${file.ftpfolder}"
filter="compositeFilterRemote"
preserve-timestamp="true"
auto-startup="true">
<int:poller fixed-rate="1000"/>
</int-ftp:inbound-channel-adapter>
<int-ftp:outbound-channel-adapter id="ftpOutbound"
channel="ftpChannel"
session-factory="ftpClientFactory"
charset="UTF-8"
remote-file-separator="/"
auto-create-directory="true"
remote-directory="${file.ftpfolder}"
use-temporary-file-name="true"
temporary-file-suffix=".writing">
<int-ftp:request-handler-advice-chain>
<bean class="org.springframework.integration.handler.advice.ExpressionEvaluatingRequestHandlerAdvice">
<property name="onSuccessExpression" value="payload.delete()" />
</bean>
</int-ftp:request-handler-advice-chain>
</int-ftp:outbound-channel-adapter>
<bean id="acceptAllFilter" class="org.springframework.integration.file.filters.AcceptAllFileListFilter" />
<bean id="compositeFilterLocal" class="org.springframework.integration.file.filters.CompositeFileListFilter">
<constructor-arg>
<list>
<!-- Ensures that the file is whole before processing it -->
<bean class="com.polling.util.CustomFileFilterLocal"/>
<!-- Ensures files are picked up only once from the directory -->
<bean class="org.springframework.integration.file.filters.AcceptAllFileListFilter" />
</list>
</constructor-arg>
</bean>
<bean id="compositeFilterRemote" class="org.springframework.integration.file.filters.CompositeFileListFilter">
<constructor-arg>
<list>
<!-- Ensures that the file is whole before processing it -->
<bean class="com.polling.util.CustomFileFilterRemote"/>
<!-- Ensures files are picked up only once from the directory -->
<bean class="org.springframework.integration.file.filters.AcceptOnceFileListFilter" />
</list>
</constructor-arg>
</bean>
If anyone has any suggestion to make this more efficient , please let me know. As of now I can repeatedly download files. So am continuing with this..
Thanks

Resources