Spring batch multiple excel streams in parallel using partitioning - multithreading

Question

I would like to use read multiple xlsx sheets in parallel using following combination
com.monitorjbl - xlsx-streamer + spring batch / partitioning
I went through the docs
https://docs.spring.io/spring-batch/4.2.x/reference/html/scalability.html#partitioning
and came up with custom ItemStreamReader implementation
#StepScope
#Component
#Scope("prototype")
public class ExcelItemReader implements ItemStreamReader<List<Object>>, ImportContext {
private Iterator<Row> rowIterator;
private Workbook workbook;
private InputStream inputStream;
private int line = 0;
#Override
public List<Object> read() {
List<Object> row = readSingleRow(this.rowIterator);
if (row != null && row.size() > 0 && row.get(0) != null) {
return row;
}
return null;
}
#Override
public void open(#NotNull ExecutionContext executionContext) throws ItemStreamException {
String fileName = (String) executionContext.get("fileName");
File file = null;
try {
file = ResourceUtils.getFile(this.getClass().getResource(fileName));
} catch (FileNotFoundException e) {
throw new ItemStreamException(e);
}
try (
InputStream inputStream = FileUtils.openInputStream(file);
Workbook workbook = StreamingReader.builder()
.rowCacheSize(5000)
.bufferSize(10000)
.open(inputStream)) {
this.inputStream = inputStream;
this.workbook = workbook;
Sheet sheet = workbook.getSheetAt(0);
this.rowIterator = sheet.rowIterator();
List<String> columns =
readSingleRow(this.rowIterator)
.stream()
.map(Object::toString)
.collect(Collectors.toList());
saveToContext(executionContext, ImportContext.headerKey, String.join(",", columns));
} catch (IOException e) {
throw new ItemStreamException(e);
}
}
#Override
public void update(#NotNull ExecutionContext executionContext) throws ItemStreamException {
}
#Override
public void close() throws ItemStreamException {
try {
this.workbook.close();
this.inputStream.close();
} catch (IOException e) {
throw new ItemStreamException(e);
}
}
private List<Object> readSingleRow(Iterator<Row> rowIterator) {
if (rowIterator.hasNext()) {
List<Object> rowContainer = new LinkedList<>();
int prevCellIndex = 0;
Row sheetRow = rowIterator.next();
for (Cell sheetCell : sheetRow) {
int currentIndex = sheetCell.getColumnIndex();
fillGaps(rowContainer, prevCellIndex, currentIndex);
rowContainer.add(sheetCell.getStringCellValue());
prevCellIndex = currentIndex;
}
line++;
return rowContainer;
}
return null;
}
private void fillGaps(List<Object> row, int prevIndex, int currentIndex) {
for (int i = prevIndex + 1; i < currentIndex; i++) {
row.add(null);
}
}
}
Configuration :
#Configuration
#EnableBatchProcessing
#EnableAutoConfiguration
#ComponentScan
public class ImportJobConfiguration<Fact extends BaseFact> {
private DataSource dataSource;
private StepBuilderFactory stepBuilderFactory;
private ImportPartitioner partitioner;
private JobExecutionListener jobListener;
private ItemWriteListener writeListener;
#Autowired
public void setDataSource(DataSource dataSource) {
this.dataSource = dataSource;
}
#Autowired
public void setStepBuilderFactory(StepBuilderFactory stepBuilderFactory) {
this.stepBuilderFactory = stepBuilderFactory;
}
#Autowired
public void setPartitioner(ImportPartitioner partitioner) {
this.partitioner = partitioner;
}
#Autowired(required = false)
public void setJobListener(JobExecutionListener jobListener) {
this.jobListener = jobListener;
}
#Autowired(required = false)
public void setItemWriteListener(ItemWriteListener writeListener) {
this.writeListener = writeListener;
}
#Autowired
public void setProcessor(FactItemProcessor<Fact> processor) {
this.processor = processor;
}
public void setFact(Fact fact) {
this.fact = fact;
}
public void setTemplate(JdbcTemplate template) {
this.template = template;
}
#Autowired
private FactItemProcessor<Fact> processor;
#Autowired
#Qualifier("sample")
private Fact fact;
#Autowired
private JdbcTemplate template;
#Bean
JdbcBatchItemWriter<Fact> writer() {
JdbcBatchItemWriter<Fact> writer = null;
try {
writer = new JdbcBatchItemWriterBuilder<Fact>()
.itemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<>())
.sql(fact.getClass().newInstance().insertSQL())
.dataSource(dataSource)
.build();
} catch (InstantiationException | IllegalAccessException e) {
e.printStackTrace();
}
return writer;
}
#Bean
public ExcelItemReader reader() {
return new ExcelItemReader();
}
#Bean
public Step partitionStep() {
return stepBuilderFactory.get("partitionStep")
.partitioner("slaveStep", partitioner)
.partitionHandler(partitionHandler())
.build();
}
#Bean
public TaskExecutor taskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setMaxPoolSize(5);
taskExecutor.setCorePoolSize(5);
taskExecutor.setQueueCapacity(10);
taskExecutor.afterPropertiesSet();
return taskExecutor;
}
#Bean
public PartitionHandler partitionHandler() {
TaskExecutorPartitionHandler retVal = new TaskExecutorPartitionHandler();
retVal.setTaskExecutor(taskExecutor());
retVal.setStep(slaveStep());
retVal.setGridSize(10);
return retVal;
}
#Bean
public Step slaveStep() {
SimpleStepBuilder<List<Object>, Fact> slaveStep = stepBuilderFactory.get("slaveStep")
.<List<Object>, Fact>chunk(100)
.reader(reader())
.processor(processor)
.writer(writer());
if (writeListener != null) {
slaveStep.listener(writeListener);
}
return slaveStep
.build();
}
#Bean
public Job createJob(#Autowired JobBuilderFactory jobBuilderFactory) {
SimpleJobBuilder jobBuilder = jobBuilderFactory.get("importJob")
.incrementer(new RunIdIncrementer())
.start(partitionStep());
if (jobListener != null) {
jobBuilder = jobBuilder
.listener(jobListener);
}
return jobBuilder.build();
}
}
Unfortunately im not able to make it thread safe. I assumed that the TaskExecutorPartitionHandler is 'assigning' Threads per partition aka per file. But it looks that the ExcelItemReader.read() is accessed randomly and i can't rely on state of the private members. Mostly the rowIterator. Any ideas and thoughts appreciated.

Answer 1

Unfortunately im not able to make it thread safe.
Since your item reader implements ItemStreamReader, you can wrap it in a SynchronizedItemStreamReader to make it thread safe:
#Bean
public SynchronizedItemStreamReader reader() throws Exception {
SynchronizedItemStreamReader synchronizedReader = new SynchronizedItemStreamReader();
synchronizedReader.setDelegate(new ExcelItemReader());
synchronizedReader.afterPropertiesSet();
return synchronizedReader;
}
You can find more details about this reader in the Item readers and writers decorators section.

Answer 2

Update:
Actually the solution almost worked. The issue was the 'try with resources' block in open method, which closes the stream somewhat randomly. I dont have explanation for this behaviour but the modified solution works as expected and im able to parse multiple excel files in paralllel and using streams.
Prototype scope on the ExcelItemReader is still important, this ensures that there are as many independent instances of ExcelItemReader how many threads you have.
The updated ExcelItemReader
import com.monitorjbl.xlsx.StreamingReader;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.bmsource.dwh.common.fileManager.FileManager;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.item.ItemStreamException;
import org.springframework.batch.item.ItemStreamReader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
#Component
#StepScope
#Scope("prototype")
public class ExcelItemReader implements ItemStreamReader<List<Object>>, ImportContext {
private Logger logger = LoggerFactory.getLogger(ExcelItemReader.class.getName());
private int rowsCount = -1;
private Iterator<Row> rowIterator;
private Workbook workbook;
private InputStream inputStream;
private ExecutionContext executionContext;
#Value("#{jobParameters['transaction']}")
private String transaction;
#Value("#{stepExecutionContext['fileName']}")
private String fileName;
#Autowired
private FileManager fileManager;
#Override
public List<Object> read() {
List<Object> row = readSingleRow(this.rowIterator);
if (row != null && row.size() > 0 && row.get(0) != null) {
return row;
}
return null;
}
#Override
public void open(#NotNull ExecutionContext executionContext) throws ItemStreamException {
this.executionContext = executionContext;
try {
InputStream inputStream = fileManager.getStream(transaction, fileName);
Workbook workbook = StreamingReader.builder()
.rowCacheSize(5000)
.bufferSize(1024)
.open(inputStream);
this.inputStream = inputStream;
this.workbook = workbook;
logger.debug("Excel file {} opened for reading", this.fileName);
Sheet sheet = workbook.getSheetAt(0);
executionContext.put(ImportContext.totalRowsKey, sheet.getLastRowNum());
executionContext.put(ImportContext.rowsKey, 0);
this.rowIterator = sheet.rowIterator();
List<String> columns =
readSingleRow(this.rowIterator)
.stream()
.map(Object::toString)
.collect(Collectors.toList());
saveToContext(executionContext, ImportContext.headerKey, String.join(",", columns));
} catch (Exception e) {
try {
this.workbook.close();
this.inputStream.close();
} catch (IOException ex) {
ex.printStackTrace();
}
throw new ItemStreamException(e);
}
}
#Override
public void update(#NotNull ExecutionContext executionContext) throws ItemStreamException {
}
#Override
public void close() throws ItemStreamException {
try {
logger.debug("Closing excel file " + this.fileName);
this.workbook.close();
this.inputStream.close();
} catch (IOException e) {
throw new ItemStreamException(e);
}
}
private List<Object> readSingleRow(Iterator<Row> rowIterator) {
if (rowIterator.hasNext()) {
List<Object> row = new LinkedList<>();
int prevCellIndex = 0;
Row sheetRow = rowIterator.next();
for (Cell sheetCell : sheetRow) {
int currentIndex = sheetCell.getColumnIndex();
fillGaps(row, prevCellIndex, currentIndex);
row.add(sheetCell.getStringCellValue());
prevCellIndex = currentIndex;
}
rowsCount++;
return row;
}
return null;
}
private void fillGaps(List<Object> row, int prevIndex, int currentIndex) {
for (int i = prevIndex + 1; i < currentIndex; i++) {
row.add(null);
}
}
}
Job configuation
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.*;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.job.builder.SimpleJobBuilder;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.launch.support.SimpleJobLauncher;
import org.springframework.batch.core.listener.JobExecutionListenerSupport;
import org.springframework.batch.core.partition.PartitionHandler;
import org.springframework.batch.core.partition.support.TaskExecutorPartitionHandler;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.batch.core.step.builder.SimpleStepBuilder;
import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
import org.springframework.batch.item.database.JdbcBatchItemWriter;
import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.SimpleAsyncTaskExecutor;
import org.springframework.core.task.TaskExecutor;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import javax.sql.DataSource;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
#Configuration
#EnableBatchProcessing
#EnableAutoConfiguration
#ComponentScan
public class ImportJobConfiguration<Fact extends BaseFact> {
private Logger logger = LoggerFactory.getLogger(ImportJobConfiguration.class);
private static final int BATCH_SIZE = 5000;
private static final int MAX_CONCURRENT_FILES = 10;
private DataSource dataSource;
private StepBuilderFactory stepBuilderFactory;
private ImportPartitioner excelImportPartitioner;
private JobExecutionListener jobListener;
private ItemWriteListener writeListener;
private ChunkListener chunkListener;
private FactItemProcessor<Fact> processor;
private Fact fact;
#Autowired
AppStateService appStateService;
#Autowired
JobRepository jobRepository;
#Autowired
public void setDataSource(DataSource dataSource) {
this.dataSource = dataSource;
}
#Autowired
public void setStepBuilderFactory(StepBuilderFactory stepBuilderFactory) {
this.stepBuilderFactory = stepBuilderFactory;
}
#Autowired
public void setExcelImportPartitioner(ImportPartitioner excelImportPartitioner) {
this.excelImportPartitioner = excelImportPartitioner;
}
#Autowired(required = false)
public void setJobListener(JobExecutionListener jobListener) {
this.jobListener = jobListener;
}
#Autowired(required = false)
public void setItemWriteListener(ItemWriteListener writeListener) {
this.writeListener = writeListener;
}
#Autowired(required = false)
public void setChunkListener(ChunkListener chunkListener) {
this.chunkListener = chunkListener;
}
#Autowired
public void setProcessor(FactItemProcessor<Fact> processor) {
this.processor = processor;
}
#Autowired
#Qualifier("fact")
public void setFact(Fact fact) {
this.fact = fact;
}
#Bean
JdbcBatchItemWriter<Fact> writer() {
JdbcBatchItemWriter<Fact> writer = null;
try {
writer = new JdbcBatchItemWriterBuilder<Fact>()
.itemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<>())
.sql(fact.getClass().newInstance().insertSQL())
.dataSource(dataSource)
.build();
} catch (InstantiationException | IllegalAccessException e) {
e.printStackTrace();
}
return writer;
}
#Bean
#StepScope
public ExcelItemReader reader() {
return new ExcelItemReader();
}
#Bean
public Step excelReadPartitionStep() {
return stepBuilderFactory.get("excelReadPartitionStep")
.partitioner("excelReadPartitioner", excelImportPartitioner)
.partitionHandler(excelReadPartitioner())
.build();
}
#Bean
public PartitionHandler excelReadPartitioner() {
TaskExecutorPartitionHandler partitionHandler = new TaskExecutorPartitionHandler();
partitionHandler.setTaskExecutor(taskExecutor());
partitionHandler.setStep(excelReadStep());
partitionHandler.setGridSize(MAX_CONCURRENT_FILES);
return partitionHandler;
}
#Bean
public Step excelReadStep() {
SimpleStepBuilder<List<Object>, Fact> step = stepBuilderFactory.get("excelReadStep")
.<List<Object>, Fact>chunk(BATCH_SIZE)
.reader(reader())
.processor(processor)
.writer(writer());
if (writeListener != null) {
step.listener(writeListener);
}
if (chunkListener != null) {
step.listener(chunkListener);
}
return step
.build();
}
#Bean
public Job importJob(#Autowired JobBuilderFactory jobBuilderFactory) {
SimpleJobBuilder jobBuilder = jobBuilderFactory.get("importJob")
//.incrementer(new RunIdIncrementer())
.start(excelReadPartitionStep());
if (jobListener != null) {
jobBuilder = jobBuilder
.listener(jobListener);
}
return jobBuilder.build();
}
#Bean
public JobLauncher simpleJobLauncher() throws Exception {
SimpleJobLauncher jobLauncher = new SimpleJobLauncher();
jobLauncher.setJobRepository(jobRepository);
jobLauncher.setTaskExecutor(new SimpleAsyncTaskExecutor());
jobLauncher.afterPropertiesSet();
return jobLauncher;
}
#Bean
public JobExecutionListener jobListener() {
JobExecutionListener listener = new JobExecutionListenerSupport() {
#Override
public void beforeJob(JobExecution jobExecution) {
}
#Override
public void afterJob(JobExecution jobExecution) {
}
};
return listener;
}
#Bean
public ChunkListener chunkListener() {
return new ChunkListener() {
#Override
public void beforeChunk(ChunkContext context) {
}
#Override
public void afterChunk(ChunkContext context) {
// some progress indicator
}
#Override
public void afterChunkError(ChunkContext context) {
}
};
}
#Bean
public TaskExecutor taskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setMaxPoolSize(MAX_CONCURRENT_FILES);
taskExecutor.setCorePoolSize(MAX_CONCURRENT_FILES / 2);
taskExecutor.setQueueCapacity(MAX_CONCURRENT_FILES / 2);
taskExecutor.afterPropertiesSet();
return taskExecutor;
}
}