Write Flowfile content into Excel (or xls) file by Groovy - excel

I am using Apache Nifi to extract data SQL query into XLS file.
I use ExecuteQuery to extract data into Avro, then to CSV using CSV RecordWriter
Now I should to have my data in an XLS file, to do, I am using this Groovy script but it's not working:
// import org.apache.commons.io.IOUtils
import java.nio.charset.*
// import java.text.SimpleDateFormat
import java.io.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.hssf.usermodel.*
import org.apache.poi.xssf.usermodel.*
import org.apache.poi.ss.util.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.hssf.extractor.*
def flowFile = session.get()
if(!flowFile) return
flowFile = session.write(flowFile, {inputStream, outputStream ->
try {
inputStream.writeTo(outputStream)
// i tried also outputStream.write(inputStream)
//i tried also to retrieve the excel file with:
//Workbook wb = WorkbookFactory.create(inputStream)
//and write with : outputStream.write(wb)
}
catch(e) {
log.error("Error during processing", e)
session.transfer(flowFile, REL_FAILURE)
}
} as StreamCallback)
session.transfer(flowFile, REL_SUCCESS)
I have this error:
Edit:
I changed my script and had an other error:
#Grapes(#Grab(group='org.apache.poi', module='poi-ooxml', version='3.9'))
import com.opencsv.CSVReader
#Grapes(#Grab(group='com.opencsv', module='opencsv', version='4.2'))
import org.apache.poi.ss.usermodel.*
import org.apache.poi.xssf.streaming.*
import org.apache.poi.hssf.usermodel.*
import org.apache.poi.xssf.usermodel.*
import org.apache.poi.ss.util.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.hssf.extractor.*
import java.nio.charset.*
import java.io.*
import org.apache.commons.io.IOUtils
def flowFile = session.get()
def date = new Date()
if(!flowFile) return
flowFile = session.write(flowFile, {inputStream, outputStream ->
SXSSFSheet sheet1 = null;
CSVReader reader = null;
Workbook wb = null;
String generatedXlsFilePath = "/home/";
FileOutputStream fileOutputStream = null;
def filename = flowFile.getAttribute('filename')
def path = flowFile.getAttribute('path')
def nextLine = ''
reader = new CSVReader(new FileReader(path+filename), ',');
//Workbook wb = WorkbookFactory.create(inputStream,);
//Sheet sheet1 = wb.createSheet("Feuille");
wb = new SXSSFWorkbook(inputStream);
sheet1 = (SXSSFSheet) wb.createSheet('Sheet');
def rowNum = 0;
while((nextLine = reader.readNext()) != null) {
Row currentRow = sheet1.createRow(rowNum++);
for(int i=0; i < nextLine.length; i++) {
if(NumberUtils.isDigits(nextLine[i])) {
currentRow.createCell(i).setCellValue(Integer.parseInt(nextLine[i]));
} else if (NumberUtils.isNumber(nextLine[i])) {
currentRow.createCell(i).setCellValue(Double.parseDouble(nextLine[i]));
} else {
currentRow.createCell(i).setCellValue(nextLine[i]);
}
}
}
//fileOutputStream = new FileOutputStream(generatedXlsFilePath.trim());
//wb.write(fileOutputStream);
generatedXlsFilePath = generatedXlsFilePath + 'SAISIE_MAGASING.XLS'
outputStream = new FileOutputStream(generatedXlsFilePath.trim());
wb.write(outputStream);
wb.close();
//fileOutputStream.close();
outputStream.close();
reader.close();
//outputStream.close();
inputStream.close();
} as StreamCallback)
flowFile = session.putAttribute(flowFile, 'filename', filename)
The new error:

Thanks to #dagget
This is the worked script:
#Grapes(#Grab(group='org.apache.poi', module='poi-ooxml', version='3.9'))
import com.opencsv.CSVReader
#Grapes(#Grab(group='com.opencsv', module='opencsv', version='4.2'))
//#Grapes(#Grab(group='org.apache.commons', module='lang', version='3.12.0'))
#Grapes(#Grab(group='commons-lang', module='commons-lang', version='2.4'))
import org.apache.commons.lang.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.xssf.streaming.*
import org.apache.poi.hssf.usermodel.*
import org.apache.poi.xssf.usermodel.*
import org.apache.poi.ss.util.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.hssf.extractor.*
import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import java.nio.charset.*
import java.io.*
import org.apache.commons.io.IOUtils
def flowFile = session.get()
def date = new Date()
if(!flowFile) return
flowFile = session.write(flowFile, {inputStream, outputStream ->
SXSSFSheet sheet1 = null;
CSVReader reader = null;
Workbook wb = null;
def nextLine = ''
reader = new CSVReader( inputStream.newReader('UTF-8') );
wb = new SXSSFWorkbook();
sheet1 = (SXSSFSheet) wb.createSheet('Sheet');
def rowNum = 0
while((nextLine = reader.readNext()) != null) {
Row currentRow = sheet1.createRow(rowNum++);
for(int i=0; i < nextLine.length; i++) {
if(NumberUtils.isDigits(nextLine[i])) {
currentRow.createCell(i).setCellValue(Integer.parseInt(nextLine[i]));
} else if (NumberUtils.isNumber(nextLine[i])) {
currentRow.createCell(i).setCellValue(Double.parseDouble(nextLine[i]));
} else {
currentRow.createCell(i).setCellValue(nextLine[i]);
}
}
}
wb.write(outputStream);
reader.close();
} as StreamCallback)
def filename = 'toto.xlsx'
flowFile = session.putAttribute(flowFile, 'filename', filename)
session.transfer(flowFile, REL_SUCCESS)

Related

Empty flowfile after trying to write content as Excel

I have following code to read .csv and transform it to Excel:
#Grab("org.apache.poi:poi:5.0.0")
#Grab("org.apache.poi:poi-ooxml:5.0.0")
#Grab("com.opencsv:opencsv:4.6")
import java.io.*;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.usermodel.*;
import org.apache.poi.xssf.streaming.*;
import org.apache.poi.ss.SpreadsheetVersion;
import org.apache.poi.ss.util.AreaReference;
import org.apache.poi.ss.util.CellReference;
import com.opencsv.CSVReader;
def flowFile = session.get()
if(!flowFile)
return
flowFile = session.write(flowFile, { inputStream, outputStream ->
try {
SXSSFWorkbook sxssfWorkbook = new SXSSFWorkbook();
CSVReader csvReader = new CSVReader(new InputStreamReader(inputStream));
sxssfWorkbook.setCompressTempFiles(true);
SXSSFSheet sxssfSheet = sxssfWorkbook.createSheet("Sheet");
sxssfSheet.setRandomAccessWindowSize(100);
String[] strHeaders = null;
String[] dataRow = null;
int rowNum = 0;
while ((dataRow = csvReader.readNext()) != null) {
if (rowNum == 0) strHeaders = dataRow;
Row currentRow = sxssfSheet.createRow(rowNum);
for (int i = 0; i < dataRow.length; i++) {
String cellValue = dataRow[i];
currentRow.createCell(i).setCellValue(cellValue);
}
rowNum++;
}
int lastRow = rowNum -1;
int lastCol = strHeaders.length -1;
AreaReference areaReference = new AreaReference(new CellReference(0, 0), new CellReference(lastRow, lastCol), SpreadsheetVersion.EXCEL2007);
XSSFWorkbook xssfWorkbook = sxssfSheet.getWorkbook().getXSSFWorkbook();
XSSFSheet xssfSheet = xssfWorkbook.getSheet(sxssfSheet.getSheetName());
XSSFTable xssfTable = xssfSheet.createTable(areaReference);
//xssfTable.updateHeaders(); // this cannot work since xssfSheet does not contain any data until now
for (int i = 0; i < strHeaders.length; i++) {
String columnHeader = strHeaders[i];
if (xssfTable.getCTTable().getTableColumns().getTableColumnList().size() > i)
xssfTable.getCTTable().getTableColumns().getTableColumnList().get(i).setName(columnHeader);
}
xssfTable.getCTTable().addNewTableStyleInfo();
XSSFTableStyleInfo style = (XSSFTableStyleInfo)xssfTable.getStyle();
style.setName("TableStyleLight9");
style.setShowColumnStripes(false);
style.setShowRowStripes(true);
xssfTable.getCTTable().addNewAutoFilter().setRef(areaReference.formatAsString());
sxssfWorkbook.write(outputStream)
} catch (Exception ex) {
ex.printStackTrace();
}
} as StreamCallback)
session.transfer(flowFile, REL_SUCCESS)
But it returns empty flowfile after ExecuteGroovyScript.
Before I tested this in IntelliJ IDEA with Java. But reading from file and saving to file:
FileOutputStream fileout = new FileOutputStream("P:\\test.xlsx");
CSVReader csvReader = new CSVReader(new FileReader("P:\\test.csv"));
So, I guess something wrong with inputstream/outputstream?
P.S. To test some values from cell I decided to write some value to attributes:
...
sxssfWorkbook.write(outputStream);
flowFile.putAttribute('test', sxssfSheet.getRow(5).getCell(1).toString())
...
But flowfile doesn't contain this attribute.
What did I wrong?

Nifi - Groovy Json Script define null attributes

I have a Nifi Groovy Script. Assigns fields to nifi attributes but it define null if json values ​​are empty. if json value is null I want to define attribute empty.
null attributes look like this ;
I want to like this ;
this is the script i use ;
import org.apache.commons.io.IOUtils
import java.nio.charset.*
def flowFile = session.get();
if (flowFile == null) {
return;
}
def slurper = new groovy.json.JsonSlurper()
def attrs = [:] as Map<String,String>
session.read(flowFile,
{ inputStream ->
def text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
def obj = slurper.parseText(text)
obj.each {k,v ->
attrs[k] = v.toString()
}
} as InputStreamCallback)
flowFile = session.putAllAttributes(flowFile, attrs)
session.transfer(flowFile, REL_SUCCESS)
flowFile = session.putAllAttributes(flowFile, attrs.collectEntries{k,v->[k,v?:'']})
This is i solved it ;
import org.apache.commons.io.IOUtils
import java.nio.charset.*
def flowFile = session.get();
if (flowFile == null) {
return;
}
def slurper = new groovy.json.JsonSlurper()
def attrs = [:] as Map<String,String>
session.read(flowFile,
{ inputStream ->
def text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
def obj = slurper.parseText(text)
obj.each {k,v ->
if (v.toString()=="null")
{
attrs[k] = ''
} else {
attrs[k] = v.toString()
}
}
} as InputStreamCallback)
flowFile = session.putAllAttributes(flowFile, attrs)
session.transfer(flowFile, REL_SUCCESS)

How to change the content of the table in the Text box of docX file

How to change the content of the table in the Text box of docX file?When xmlcursor.getobject () instanceof XmlAnyTypeImpl is instanceof XmlAnyTypeImpl, I cannot change the contents of the table in the text box. How can I change the contents of the table in the text box?Here's my code, but he can't change it properly。
import java.io.*;
import org.apache.poi.xwpf.usermodel.*;
import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTBody;
import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTbl;
import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTRow;
import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTc;
import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTP;
import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTR;
import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTText;
import org.apache.xmlbeans.impl.values.XmlAnyTypeImpl;
import org.apache.xmlbeans.XmlCursor;
import javax.xml.namespace.QName;
import java.util.List;
import java.util.ArrayList;
public class WordReadAllTables {
public static void main(String[] args) throws Exception {
XWPFDocument document = new XWPFDocument(new FileInputStream(System.getProperty("user.dir") + "\\template\\" + "test.docx"));
CTBody ctbody = document.getDocument().getBody();
XmlCursor xmlcursor = ctbody.newCursor();
QName qnameTbl = new QName("http://schemas.openxmlformats.org/wordprocessingml/2006/main", "tbl", "w");
QName qnameFallback = new QName("http://schemas.openxmlformats.org/markup-compatibility/2006", "Fallback", "mc");
List<CTTbl> allCTTbls = new ArrayList<CTTbl>();
while (xmlcursor.hasNextToken()) {
XmlCursor.TokenType tokentype = xmlcursor.toNextToken();
if (tokentype.isStart()) {
if (qnameTbl.equals(xmlcursor.getName())) {
if (xmlcursor.getObject() instanceof CTTbl) {
allCTTbls.add((CTTbl) xmlcursor.getObject());
} else if (xmlcursor.getObject() instanceof XmlAnyTypeImpl) {
allCTTbls.add(CTTbl.Factory.parse(xmlcursor.getObject().toString()));
}
} else if (qnameFallback.equals(xmlcursor.getName())) {
xmlcursor.toEndToken();
}
}
}
for (CTTbl cTTbl : allCTTbls) {
StringBuffer tableHTML = new StringBuffer();
tableHTML.append("<table>\n");
for (CTRow cTRow : cTTbl.getTrList()) {
for (CTTc cTTc : cTRow.getTcList()) {
for (CTP cTP : cTTc.getPList()) {
for (CTR cTR : cTP.getRList()) {
for (CTText cTText : cTR.getTList()) {
tableHTML.append(cTText.getStringValue() + "-");
if (cTText.getStringValue().contains("select")) {
cTText.setStringValue("1111");
}
}
}
}
}
}
}
FileOutputStream stream = new FileOutputStream(new File("D://aa.docx"));
document.write(stream);
stream.close();
document.close();
}
}

Nifi: convert xml: bug with groovy script

I need covert xml files in Nifi 1.5. I try execute this script in ExcecuteScript processor:
import groovy.xml.*
import javax.xml.transform.TransformerFactory
import javax.xml.transform.Transformer
import javax.xml.transform.dom.DOMSource
import javax.xml.transform.stream.StreamResult
import org.apache.commons.io.IOUtils
import java.nio.charset.StandardCharsets
import org.apache.nifi.processor.io.StreamCallback
import java.io.StringWriter
flowFile = session.get()
if(!flowFile) return
session.write(flowFile,
{
inputStream, outputStream ->
def xml = IOUtils.toString(inputStream, 'UTF-8')
def reader = new StringReader(xml)
def doc = DOMBuilder.parse(reader)
def records = doc.documentElement
TransformerFactory tFactory = TransformerFactory.newInstance();
StringWriter writer = new StringWriter();
Transformer transformer = tFactory.newTransformer();
DOMSource source = new DOMSource(records);
StreamResult result = new StreamResult(writer);
String strString = writer.toString()
outputStream.write(strString.getBytes(StandardCharsets.UTF_8))
} as StreamCallback
)
session.transfer(flowFile, REL_SUCCESS)
And Output FlowFile is empty.
In local this part working:
def xml = "<xml>...</xml>"
def reader = new StringReader(xml)
def doc = DOMBuilder.parse(reader)
def records = doc.documentElement
TransformerFactory tFactory = TransformerFactory.newInstance();
StringWriter writer = new StringWriter();
Transfor mer transformer = tFactory.newTransformer();
DOMSource source = new DOMSource(records);
StreamResult result = new StreamResult(writer);
String strString = writer.toString()
Why output file is empty?
I missed the line transformer.transform(source, result)
full script for Nifi:
import groovy.xml.*
import javax.xml.transform.TransformerFactory
import javax.xml.transform.Transformer
import javax.xml.transform.dom.DOMSource
import javax.xml.transform.stream.StreamResult
import org.apache.commons.io.IOUtils
import java.nio.charset.StandardCharsets
import org.apache.nifi.processor.io.StreamCallback
import java.io.StringWriter
flowFile = session.get()
if(!flowFile) return
session.write(flowFile,
{
inputStream, outputStream ->
def xml = IOUtils.toString(inputStream, 'UTF-8')
def reader = new StringReader(xml)
def doc = DOMBuilder.parse(reader)
def records = doc.documentElement
TransformerFactory tFactory = TransformerFactory.newInstance();
StringWriter writer = new StringWriter();
Transformer transformer = tFactory.newTransformer();
DOMSource source = new DOMSource(records);
StreamResult result = new StreamResult(writer);
transformer.transform(source, result)
String strString = writer.toString()
outputStream.write(strString.getBytes(StandardCharsets.UTF_8))
} as StreamCallback
)
session.transfer(flowFile, REL_SUCCESS)

Selenium IDE: read set of data from excel

This is actually not a question but solution proposal: I found workaround to read set of data from excel. in this case there is no need for multiple users or data variation but read parameters to create a validation environment.
ok, solution is to save excel file to html format and then let the Selenium IDE to read parameters from that. Users needs only to agree the same filename to be used.
1) you should add "Apache POI" jar files in order to read your excel through java.
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFRow;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
public class ExcelReadExample {
#SuppressWarnings({ "unchecked", "rawtypes" })
public static void main(String[] args) throws Exception {
String filename = "E:\\data.xls";
List sheetData = new ArrayList();
FileInputStream fis = null;
try {
fis = new FileInputStream(filename);
XSSFWorkbook workbook = new XSSFWorkbook(fis);
XSSFSheet sheet = workbook.getSheetAt(0);
Iterator rows = sheet.rowIterator();
while (rows.hasNext()) {
XSSFRow row = (XSSFRow) rows.next();
Iterator cells = row.cellIterator();
List data = new ArrayList();
while (cells.hasNext()) {
XSSFCell cell = (XSSFCell) cells.next();
data.add(cell);
}
sheetData.add(data);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (fis != null) {
fis.close();
}
}
showExelData(sheetData);
}
private static void showExelData(List sheetData) {
int sum = 0;
for (int i = 0; i < sheetData.size(); i++) {
List<XSSFCell> list = (List) sheetData.get(i);
for (int j = 0; j < list.size(); j++) {
XSSFCell cell = (XSSFCell) list.get(j);
if(cell.getCellType()==0)
{
sum += cell.getNumericCellValue();
}
}
System.out.println("");
System.out.println("Sum Value is:" +sum);
}
}
}
Change the file path.
i hve mentioned my sheet name as "input" change it as per yours
Happy excelling :D

Resources