Handling a method in integration flow - spring-integration

Question

I have created an application that uses Spring MVC to add ftp servers through a web page. When I add a server with details, it will start a flow polling a directory on the sever for specific csv file, once that file is there it will pull it to my local folder then a method is created to read that csv file and generate a new csv that will be sent back to the server, now this is accomplished, my issue is that when I add a new ftp server to poll and it finds a csv file on the second server, the application is pulling that file to my local as it should do but the handling method is being triggered again for both csv files while it should only do so for the second server since it is already triggered for the first server, so when I add the second server then again another file for server 1 generated and sent back to ftp and new one for the second server is send to the second ftp, at the end I have two files sent to first serve and one file to the second server, below is the coding and the output consol of the app, please assist on pinpointing my issue.
#Configuration
#EnableIntegration
#ComponentScan
public class FTIntegration {
public static final String TIMEZONE_UTC = "UTC";
public static final String TIMESTAMP_FORMAT_OF_FILES = "yyyyMMddHHmmssSSS";
public static final String TEMPORARY_FILE_SUFFIX = ".part";
public static final int POLLER_FIXED_PERIOD_DELAY = 5000;
public static final int MAX_MESSAGES_PER_POLL = 100;
//private static final Logger LOG = LoggerFactory.getLogger(FTIntegration.class);
private static final Logger LOG1 = Logger.getLogger(FTIntegration.class);
private static final String CHANNEL_INTERMEDIATE_STAGE = "intermediateChannel";
private static final String OUTBOUND_CHANNEL = "outboundChannel";
/* pulling the server config from postgres DB*/
private final BranchRepository branchRepository;
#Autowired
private CSVToCSVNoQ csvToCSVNoQ;
#Value("${app.temp-dir}")
private String localTempPath;
public FTIntegration(BranchRepository branchRepository) {
this.branchRepository = branchRepository;
}
#Bean
public Branch myBranch(){
return new Branch();
}
/**
* The default poller with 5s, 100 messages, RotatingServerAdvice and transaction.
*
* #return default poller.
*/
#Bean(name = PollerMetadata.DEFAULT_POLLER)
public PollerMetadata poller(){
return Pollers
.fixedDelay(POLLER_FIXED_PERIOD_DELAY)
.maxMessagesPerPoll(MAX_MESSAGES_PER_POLL)
.transactional()
.get();
}
/**
* The direct channel for the flow.
*
* #return MessageChannel
*/
#Bean
public MessageChannel stockIntermediateChannel() {
return new DirectChannel();
}
/**
* Get the files from a remote directory. Add a timestamp to the filename
* and write them to a local temporary folder.
*
* #return IntegrationFlow
*/
#Bean
public PropertiesPersistingMetadataStore store() {
PropertiesPersistingMetadataStore store = new PropertiesPersistingMetadataStore();
return store;
}
public IntegrationFlow fileInboundFlowFromFTPServer(Branch myBranch) throws IOException {
final FtpInboundChannelAdapterSpec sourceSpecFtp = Ftp.inboundAdapter(createNewFtpSessionFactory(myBranch))
.preserveTimestamp(true)
//.patternFilter("*.csv")
.maxFetchSize(MAX_MESSAGES_PER_POLL)
.remoteDirectory(myBranch.getFolderPath())
.regexFilter("FEFOexport"+myBranch.getBranchCode()+".csv")
.deleteRemoteFiles(true)
.localDirectory(new File(myBranch.getBranchCode()))
.temporaryFileSuffix(TEMPORARY_FILE_SUFFIX)
/*.localFilenameExpression(new FunctionExpression<String>(s -> {
final int fileTypeSepPos = s.lastIndexOf('.');
return DateTimeFormatter
.ofPattern(TIMESTAMP_FORMAT_OF_FILES)
.withZone(ZoneId.of(TIMEZONE_UTC))
.format(Instant.now())
+ "_"
+ s.substring(0,fileTypeSepPos)
+ s.substring(fileTypeSepPos);
}))*/;
// Poller definition
final Consumer<SourcePollingChannelAdapterSpec> stockInboundPoller = endpointConfigurer -> endpointConfigurer
.id("stockInboundPoller")
.autoStartup(true)
.poller(poller());
IntegrationFlow flow = IntegrationFlows
.from(sourceSpecFtp, stockInboundPoller)
.transform(File.class, p ->{
// log step
LOG1.info("flow=stockInboundFlowFromAFT, message=incoming file: " + p);
return p;
})
.channel(CHANNEL_INTERMEDIATE_STAGE)
.handle(m -> {
try {
this.csvToCSVNoQ.writeCSVfinal("test", myBranch.getBranchCode() + "/final" + myBranch.getBranchCode() + ".csv", myBranch.getBranchCode() + "/FEFOexport" + myBranch.getBranchCode() + ".csv");
LOG1.info("Writing final file .csv " + m);
} catch (IOException e) {
e.printStackTrace();
}
})
//.handle(m -> this.csvToCSVNoQ.writeCSVfinal(m.getPayload(),m.getHeaders().get("csv", "FEFOexport"+myBranch.getBranchCode()+".csv")))
.get();
return flow;
}
#Bean
public IntegrationFlow stockIntermediateStageChannel() {
IntegrationFlow flow = IntegrationFlows
.from(CHANNEL_INTERMEDIATE_STAGE)
.transform(p -> {
//log step
LOG1.info("flow=stockIntermediateStageChannel, message=rename file: " + p);
return p;
})
//TODO
.channel(new NullChannel())
.get();
return flow;
}
/*
* Creating the outbound adaptor to send files from local to FTP server
*
* */
public IntegrationFlow localToFtpFlow(Branch myBranch){
return IntegrationFlows.from(Files.inboundAdapter(new File(myBranch.getBranchCode()))
.filter(new ChainFileListFilter<File>()
.addFilter(new RegexPatternFileListFilter("final" + myBranch.getBranchCode() +".csv"))
.addFilter(new FileSystemPersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "foo"))),
e -> e.poller(Pollers.fixedDelay(10_000)))
.transform( p ->{
LOG1.info("Sending file " + p + " to FTP branch " + myBranch.getBranchCode());
return p;
})
.log()
.handle(Ftp.outboundAdapter(createNewFtpSessionFactory(myBranch),FileExistsMode.REPLACE)
.useTemporaryFileName(true)
.autoCreateDirectory(false)
.remoteDirectory(myBranch.getFolderPath()))
.get();
}
public DefaultFtpSessionFactory createNewFtpSessionFactory(Branch branch){
final DefaultFtpSessionFactory factory = new DefaultFtpSessionFactory();
factory.setHost(branch.getHost());
factory.setUsername(branch.getUsern());
factory.setPort(branch.getFtpPort());
factory.setPassword(branch.getPassword());
return factory;
}
Console Output:
Saved Branch : BEY
Hibernate: select branch0_._id as _id1_0_0_, branch0_.branch_code as branch_c2_0_0_, branch0_.folder_path as folder_p3_0_0_, branch0_.ftp_port as ftp_port4_0_0_, branch0_.host as host5_0_0_, branch0_.password as password6_0_0_, branch0_.usern as usern7_0_0_ from branch branch0_ where branch0_._id=?
2018-12-20 07:58:39.218 INFO 6668 --- [nio-8081-exec-5] o.s.integration.channel.DirectChannel : Channel 'application.intermediateChannel' has 2 subscriber(s).
2018-12-20 07:58:39.218 INFO 6668 --- [nio-8081-exec-5] o.s.i.endpoint.EventDrivenConsumer : started 1.org.springframework.integration.config.ConsumerEndpointFactoryBean#1
2018-12-20 07:58:39.218 INFO 6668 --- [nio-8081-exec-5] o.s.i.endpoint.EventDrivenConsumer : Adding {transformer} as a subscriber to the '1.channel#0' channel
2018-12-20 07:58:39.218 INFO 6668 --- [nio-8081-exec-5] o.s.integration.channel.DirectChannel : Channel 'application.1.channel#0' has 1 subscriber(s).
2018-12-20 07:58:39.218 INFO 6668 --- [nio-8081-exec-5] o.s.i.endpoint.EventDrivenConsumer : started 1.org.springframework.integration.config.ConsumerEndpointFactoryBean#0
2018-12-20 07:58:39.229 INFO 6668 --- [nio-8081-exec-5] o.s.i.e.SourcePollingChannelAdapter : started stockInboundPoller
2018-12-20 07:58:39.417 INFO 6668 --- [nio-8081-exec-5] o.s.i.endpoint.EventDrivenConsumer : Adding {message-handler} as a subscriber to the '1o.channel#2' channel
2018-12-20 07:58:39.417 INFO 6668 --- [nio-8081-exec-5] o.s.integration.channel.DirectChannel : Channel 'application.1o.channel#2' has 1 subscriber(s).
2018-12-20 07:58:39.418 INFO 6668 --- [nio-8081-exec-5] o.s.i.endpoint.EventDrivenConsumer : started 1o.org.springframework.integration.config.ConsumerEndpointFactoryBean#1
2018-12-20 07:58:39.418 INFO 6668 --- [nio-8081-exec-5] o.s.i.endpoint.EventDrivenConsumer : Adding {transformer} as a subscriber to the '1o.channel#0' channel
2018-12-20 07:58:39.418 INFO 6668 --- [nio-8081-exec-5] o.s.integration.channel.DirectChannel : Channel 'application.1o.channel#0' has 1 subscriber(s).
2018-12-20 07:58:39.418 INFO 6668 --- [nio-8081-exec-5] o.s.i.endpoint.EventDrivenConsumer : started 1o.org.springframework.integration.config.ConsumerEndpointFactoryBean#0
2018-12-20 07:58:39.419 INFO 6668 --- [nio-8081-exec-5] o.s.i.e.SourcePollingChannelAdapter : started 1o.org.springframework.integration.config.SourcePollingChannelAdapterFactoryBean#0
2018-12-20 07:59:41.165 INFO 6668 --- [sk-scheduler-10] o.s.integration.ftp.session.FtpSession : File has been successfully transferred from: /ftp/erbranch/EDMS/FEFO/FEFOexportBEY.csv
2018-12-20 07:59:49.446 INFO 6668 --- [ask-scheduler-1] o.s.i.file.FileReadingMessageSource : Created message: [GenericMessage [payload=BEY\finalBEY.csv, headers={file_originalFile=BEY\finalBEY.csv, id=7f0cccb7-a070-bd4c-d468-977a265ceb2e, file_name=finalBEY.csv, file_relativePath=finalBEY.csv, timestamp=1545285589446}]]
2018-12-20 07:59:49.448 INFO 6668 --- [ask-scheduler-1] o.s.integration.handler.LoggingHandler : GenericMessage [payload=BEY\finalBEY.csv, headers={file_originalFile=BEY\finalBEY.csv, id=d857130a-d4a0-eaeb-19ea-f819924d94e2, file_name=finalBEY.csv, file_relativePath=finalBEY.csv, timestamp=1545285589447}]
2018-12-20 07:59:50.488 INFO 6668 --- [ask-scheduler-1] o.s.integration.ftp.session.FtpSession : File has been successfully transferred to: /ftp/erbranch/EDMS/FEFO/finalBEY.csv.writing
2018-12-20 07:59:50.899 INFO 6668 --- [ask-scheduler-1] o.s.integration.ftp.session.FtpSession : File has been successfully renamed from: /ftp/erbranch/EDMS/FEFO/finalBEY.csv.writing to /ftp/erbranch/EDMS/FEFO/finalBEY.csv
Hibernate: select branch0_._id as _id1_0_, branch0_.branch_code as branch_c2_0_, branch0_.folder_path as folder_p3_0_, branch0_.ftp_port as ftp_port4_0_, branch0_.host as host5_0_, branch0_.password as password6_0_, branch0_.usern as usern7_0_ from branch branch0_
Hibernate: insert into branch (branch_code, folder_path, ftp_port, host, password, usern) values (?, ?, ?, ?, ?, ?)
Hibernate: select currval('branch__id_seq')
Saved Branch : JNB
Hibernate: select branch0_._id as _id1_0_0_, branch0_.branch_code as branch_c2_0_0_, branch0_.folder_path as folder_p3_0_0_, branch0_.ftp_port as ftp_port4_0_0_, branch0_.host as host5_0_0_, branch0_.password as password6_0_0_, branch0_.usern as usern7_0_0_ from branch branch0_ where branch0_._id=?
2018-12-20 08:02:24.966 INFO 6668 --- [nio-8081-exec-8] o.s.integration.channel.DirectChannel : Channel 'application.intermediateChannel' has 3 subscriber(s).
2018-12-20 08:02:24.966 INFO 6668 --- [nio-8081-exec-8] o.s.i.endpoint.EventDrivenConsumer : started 2.org.springframework.integration.config.ConsumerEndpointFactoryBean#1
2018-12-20 08:02:24.966 INFO 6668 --- [nio-8081-exec-8] o.s.i.endpoint.EventDrivenConsumer : Adding {transformer} as a subscriber to the '2.channel#0' channel
2018-12-20 08:02:24.966 INFO 6668 --- [nio-8081-exec-8] o.s.integration.channel.DirectChannel : Channel 'application.2.channel#0' has 1 subscriber(s).
2018-12-20 08:02:24.966 INFO 6668 --- [nio-8081-exec-8] o.s.i.endpoint.EventDrivenConsumer : started 2.org.springframework.integration.config.ConsumerEndpointFactoryBean#0
2018-12-20 08:02:24.966 INFO 6668 --- [nio-8081-exec-8] o.s.i.e.SourcePollingChannelAdapter : started stockInboundPoller
2018-12-20 08:02:24.992 INFO 6668 --- [nio-8081-exec-8] o.s.i.endpoint.EventDrivenConsumer : Adding {message-handler} as a subscriber to the '2o.channel#2' channel
2018-12-20 08:02:24.992 INFO 6668 --- [nio-8081-exec-8] o.s.integration.channel.DirectChannel : Channel 'application.2o.channel#2' has 1 subscriber(s).
2018-12-20 08:02:24.992 INFO 6668 --- [nio-8081-exec-8] o.s.i.endpoint.EventDrivenConsumer : started 2o.org.springframework.integration.config.ConsumerEndpointFactoryBean#1
2018-12-20 08:02:24.992 INFO 6668 --- [nio-8081-exec-8] o.s.i.endpoint.EventDrivenConsumer : Adding {transformer} as a subscriber to the '2o.channel#0' channel
2018-12-20 08:02:24.992 INFO 6668 --- [nio-8081-exec-8] o.s.integration.channel.DirectChannel : Channel 'application.2o.channel#0' has 1 subscriber(s).
2018-12-20 08:02:24.992 INFO 6668 --- [nio-8081-exec-8] o.s.i.endpoint.EventDrivenConsumer : started 2o.org.springframework.integration.config.ConsumerEndpointFactoryBean#0
2018-12-20 08:02:24.992 INFO 6668 --- [nio-8081-exec-8] o.s.i.e.SourcePollingChannelAdapter : started 2o.org.springframework.integration.config.SourcePollingChannelAdapterFactoryBean#0
Hibernate: select branch0_._id as _id1_0_, branch0_.branch_code as branch_c2_0_, branch0_.folder_path as folder_p3_0_, branch0_.ftp_port as ftp_port4_0_, branch0_.host as host5_0_, branch0_.password as password6_0_, branch0_.usern as usern7_0_ from branch branch0_
2018-12-20 08:03:00.225 INFO 6668 --- [ask-scheduler-8] o.s.integration.ftp.session.FtpSession : File has been successfully transferred from: /ftp/erbranch/EDMS/FEFO/FEFOexportJNB.csv
2018-12-20 08:03:00.929 INFO 6668 --- [ask-scheduler-5] o.s.i.file.FileReadingMessageSource : Created message: [GenericMessage [payload=BEY\finalBEY.csv, headers={file_originalFile=BEY\finalBEY.csv, id=6ed554eb-f553-0293-f042-d633155357c0, file_name=finalBEY.csv, file_relativePath=finalBEY.csv, timestamp=1545285780929}]]
2018-12-20 08:03:00.930 INFO 6668 --- [ask-scheduler-5] o.s.integration.handler.LoggingHandler : GenericMessage [payload=BEY\finalBEY.csv, headers={file_originalFile=BEY\finalBEY.csv, id=b2d76ac5-fb85-1313-a37e-8849714a545e, file_name=finalBEY.csv, file_relativePath=finalBEY.csv, timestamp=1545285780930}]
2018-12-20 08:03:01.958 INFO 6668 --- [ask-scheduler-5] o.s.integration.ftp.session.FtpSession : File has been successfully transferred to: /ftp/erbranch/EDMS/FEFO/finalBEY.csv.writing
2018-12-20 08:03:02.373 INFO 6668 --- [ask-scheduler-5] o.s.integration.ftp.session.FtpSession : File has been successfully renamed from: /ftp/erbranch/EDMS/FEFO/finalBEY.csv.writing to /ftp/erbranch/EDMS/FEFO/finalBEY.csv
2018-12-20 08:03:05.033 INFO 6668 --- [ask-scheduler-7] o.s.i.file.FileReadingMessageSource : Created message: [GenericMessage [payload=JNB\finalJNB.csv, headers={file_originalFile=JNB\finalJNB.csv, id=4514e132-5684-9e82-28e7-f75c5c3dcf91, file_name=finalJNB.csv, file_relativePath=finalJNB.csv, timestamp=1545285785033}]]
2018-12-20 08:03:05.034 INFO 6668 --- [ask-scheduler-7] o.s.integration.handler.LoggingHandler : GenericMessage [payload=JNB\finalJNB.csv, headers={file_originalFile=JNB\finalJNB.csv, id=59e62375-f1da-461d-ee61-d105ac3159a0, file_name=finalJNB.csv, file_relativePath=finalJNB.csv, timestamp=1545285785034}]
2018-12-20 08:03:07.530 INFO 6668 --- [ask-scheduler-7] o.s.integration.ftp.session.FtpSession : File has been successfully transferred to: /ftp/erbranch/EDMS/FEFO/finalJNB.csv.writing
2018-12-20 08:03:08.539 INFO 6668 --- [ask-scheduler-7] o.s.integration.ftp.session.FtpSession : File has been successfully renamed from: /ftp/erbranch/EDMS/FEFO/finalJNB.csv.writing to /ftp/erbranch/EDMS/FEFO/finalJNB.csv

Answer 1

.addFilter(new FileSystemPersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "foo"))),
The SimpleMetadataStore() is in-memory - using a new store for each flow means it starts empty each time - you need to use a shared metadata store across all the flows.
Also, if you need to prevent duplicates across server restarts, you need to use a persistent shared store, e.g. Redis, JDBC etc.