Nifi: convert xml: bug with groovy script - groovy

I need covert xml files in Nifi 1.5. I try execute this script in ExcecuteScript processor:
import groovy.xml.*
import javax.xml.transform.TransformerFactory
import javax.xml.transform.Transformer
import javax.xml.transform.dom.DOMSource
import javax.xml.transform.stream.StreamResult
import org.apache.commons.io.IOUtils
import java.nio.charset.StandardCharsets
import org.apache.nifi.processor.io.StreamCallback
import java.io.StringWriter
flowFile = session.get()
if(!flowFile) return
session.write(flowFile,
{
inputStream, outputStream ->
def xml = IOUtils.toString(inputStream, 'UTF-8')
def reader = new StringReader(xml)
def doc = DOMBuilder.parse(reader)
def records = doc.documentElement
TransformerFactory tFactory = TransformerFactory.newInstance();
StringWriter writer = new StringWriter();
Transformer transformer = tFactory.newTransformer();
DOMSource source = new DOMSource(records);
StreamResult result = new StreamResult(writer);
String strString = writer.toString()
outputStream.write(strString.getBytes(StandardCharsets.UTF_8))
} as StreamCallback
)
session.transfer(flowFile, REL_SUCCESS)
And Output FlowFile is empty.
In local this part working:
def xml = "<xml>...</xml>"
def reader = new StringReader(xml)
def doc = DOMBuilder.parse(reader)
def records = doc.documentElement
TransformerFactory tFactory = TransformerFactory.newInstance();
StringWriter writer = new StringWriter();
Transfor mer transformer = tFactory.newTransformer();
DOMSource source = new DOMSource(records);
StreamResult result = new StreamResult(writer);
String strString = writer.toString()
Why output file is empty?

I missed the line transformer.transform(source, result)
full script for Nifi:
import groovy.xml.*
import javax.xml.transform.TransformerFactory
import javax.xml.transform.Transformer
import javax.xml.transform.dom.DOMSource
import javax.xml.transform.stream.StreamResult
import org.apache.commons.io.IOUtils
import java.nio.charset.StandardCharsets
import org.apache.nifi.processor.io.StreamCallback
import java.io.StringWriter
flowFile = session.get()
if(!flowFile) return
session.write(flowFile,
{
inputStream, outputStream ->
def xml = IOUtils.toString(inputStream, 'UTF-8')
def reader = new StringReader(xml)
def doc = DOMBuilder.parse(reader)
def records = doc.documentElement
TransformerFactory tFactory = TransformerFactory.newInstance();
StringWriter writer = new StringWriter();
Transformer transformer = tFactory.newTransformer();
DOMSource source = new DOMSource(records);
StreamResult result = new StreamResult(writer);
transformer.transform(source, result)
String strString = writer.toString()
outputStream.write(strString.getBytes(StandardCharsets.UTF_8))
} as StreamCallback
)
session.transfer(flowFile, REL_SUCCESS)

Related

Empty flowfile after trying to write content as Excel

I have following code to read .csv and transform it to Excel:
#Grab("org.apache.poi:poi:5.0.0")
#Grab("org.apache.poi:poi-ooxml:5.0.0")
#Grab("com.opencsv:opencsv:4.6")
import java.io.*;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.usermodel.*;
import org.apache.poi.xssf.streaming.*;
import org.apache.poi.ss.SpreadsheetVersion;
import org.apache.poi.ss.util.AreaReference;
import org.apache.poi.ss.util.CellReference;
import com.opencsv.CSVReader;
def flowFile = session.get()
if(!flowFile)
return
flowFile = session.write(flowFile, { inputStream, outputStream ->
try {
SXSSFWorkbook sxssfWorkbook = new SXSSFWorkbook();
CSVReader csvReader = new CSVReader(new InputStreamReader(inputStream));
sxssfWorkbook.setCompressTempFiles(true);
SXSSFSheet sxssfSheet = sxssfWorkbook.createSheet("Sheet");
sxssfSheet.setRandomAccessWindowSize(100);
String[] strHeaders = null;
String[] dataRow = null;
int rowNum = 0;
while ((dataRow = csvReader.readNext()) != null) {
if (rowNum == 0) strHeaders = dataRow;
Row currentRow = sxssfSheet.createRow(rowNum);
for (int i = 0; i < dataRow.length; i++) {
String cellValue = dataRow[i];
currentRow.createCell(i).setCellValue(cellValue);
}
rowNum++;
}
int lastRow = rowNum -1;
int lastCol = strHeaders.length -1;
AreaReference areaReference = new AreaReference(new CellReference(0, 0), new CellReference(lastRow, lastCol), SpreadsheetVersion.EXCEL2007);
XSSFWorkbook xssfWorkbook = sxssfSheet.getWorkbook().getXSSFWorkbook();
XSSFSheet xssfSheet = xssfWorkbook.getSheet(sxssfSheet.getSheetName());
XSSFTable xssfTable = xssfSheet.createTable(areaReference);
//xssfTable.updateHeaders(); // this cannot work since xssfSheet does not contain any data until now
for (int i = 0; i < strHeaders.length; i++) {
String columnHeader = strHeaders[i];
if (xssfTable.getCTTable().getTableColumns().getTableColumnList().size() > i)
xssfTable.getCTTable().getTableColumns().getTableColumnList().get(i).setName(columnHeader);
}
xssfTable.getCTTable().addNewTableStyleInfo();
XSSFTableStyleInfo style = (XSSFTableStyleInfo)xssfTable.getStyle();
style.setName("TableStyleLight9");
style.setShowColumnStripes(false);
style.setShowRowStripes(true);
xssfTable.getCTTable().addNewAutoFilter().setRef(areaReference.formatAsString());
sxssfWorkbook.write(outputStream)
} catch (Exception ex) {
ex.printStackTrace();
}
} as StreamCallback)
session.transfer(flowFile, REL_SUCCESS)
But it returns empty flowfile after ExecuteGroovyScript.
Before I tested this in IntelliJ IDEA with Java. But reading from file and saving to file:
FileOutputStream fileout = new FileOutputStream("P:\\test.xlsx");
CSVReader csvReader = new CSVReader(new FileReader("P:\\test.csv"));
So, I guess something wrong with inputstream/outputstream?
P.S. To test some values from cell I decided to write some value to attributes:
...
sxssfWorkbook.write(outputStream);
flowFile.putAttribute('test', sxssfSheet.getRow(5).getCell(1).toString())
...
But flowfile doesn't contain this attribute.
What did I wrong?

Write Flowfile content into Excel (or xls) file by Groovy

I am using Apache Nifi to extract data SQL query into XLS file.
I use ExecuteQuery to extract data into Avro, then to CSV using CSV RecordWriter
Now I should to have my data in an XLS file, to do, I am using this Groovy script but it's not working:
// import org.apache.commons.io.IOUtils
import java.nio.charset.*
// import java.text.SimpleDateFormat
import java.io.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.hssf.usermodel.*
import org.apache.poi.xssf.usermodel.*
import org.apache.poi.ss.util.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.hssf.extractor.*
def flowFile = session.get()
if(!flowFile) return
flowFile = session.write(flowFile, {inputStream, outputStream ->
try {
inputStream.writeTo(outputStream)
// i tried also outputStream.write(inputStream)
//i tried also to retrieve the excel file with:
//Workbook wb = WorkbookFactory.create(inputStream)
//and write with : outputStream.write(wb)
}
catch(e) {
log.error("Error during processing", e)
session.transfer(flowFile, REL_FAILURE)
}
} as StreamCallback)
session.transfer(flowFile, REL_SUCCESS)
I have this error:
Edit:
I changed my script and had an other error:
#Grapes(#Grab(group='org.apache.poi', module='poi-ooxml', version='3.9'))
import com.opencsv.CSVReader
#Grapes(#Grab(group='com.opencsv', module='opencsv', version='4.2'))
import org.apache.poi.ss.usermodel.*
import org.apache.poi.xssf.streaming.*
import org.apache.poi.hssf.usermodel.*
import org.apache.poi.xssf.usermodel.*
import org.apache.poi.ss.util.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.hssf.extractor.*
import java.nio.charset.*
import java.io.*
import org.apache.commons.io.IOUtils
def flowFile = session.get()
def date = new Date()
if(!flowFile) return
flowFile = session.write(flowFile, {inputStream, outputStream ->
SXSSFSheet sheet1 = null;
CSVReader reader = null;
Workbook wb = null;
String generatedXlsFilePath = "/home/";
FileOutputStream fileOutputStream = null;
def filename = flowFile.getAttribute('filename')
def path = flowFile.getAttribute('path')
def nextLine = ''
reader = new CSVReader(new FileReader(path+filename), ',');
//Workbook wb = WorkbookFactory.create(inputStream,);
//Sheet sheet1 = wb.createSheet("Feuille");
wb = new SXSSFWorkbook(inputStream);
sheet1 = (SXSSFSheet) wb.createSheet('Sheet');
def rowNum = 0;
while((nextLine = reader.readNext()) != null) {
Row currentRow = sheet1.createRow(rowNum++);
for(int i=0; i < nextLine.length; i++) {
if(NumberUtils.isDigits(nextLine[i])) {
currentRow.createCell(i).setCellValue(Integer.parseInt(nextLine[i]));
} else if (NumberUtils.isNumber(nextLine[i])) {
currentRow.createCell(i).setCellValue(Double.parseDouble(nextLine[i]));
} else {
currentRow.createCell(i).setCellValue(nextLine[i]);
}
}
}
//fileOutputStream = new FileOutputStream(generatedXlsFilePath.trim());
//wb.write(fileOutputStream);
generatedXlsFilePath = generatedXlsFilePath + 'SAISIE_MAGASING.XLS'
outputStream = new FileOutputStream(generatedXlsFilePath.trim());
wb.write(outputStream);
wb.close();
//fileOutputStream.close();
outputStream.close();
reader.close();
//outputStream.close();
inputStream.close();
} as StreamCallback)
flowFile = session.putAttribute(flowFile, 'filename', filename)
The new error:
Thanks to #dagget
This is the worked script:
#Grapes(#Grab(group='org.apache.poi', module='poi-ooxml', version='3.9'))
import com.opencsv.CSVReader
#Grapes(#Grab(group='com.opencsv', module='opencsv', version='4.2'))
//#Grapes(#Grab(group='org.apache.commons', module='lang', version='3.12.0'))
#Grapes(#Grab(group='commons-lang', module='commons-lang', version='2.4'))
import org.apache.commons.lang.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.xssf.streaming.*
import org.apache.poi.hssf.usermodel.*
import org.apache.poi.xssf.usermodel.*
import org.apache.poi.ss.util.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.hssf.extractor.*
import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import java.nio.charset.*
import java.io.*
import org.apache.commons.io.IOUtils
def flowFile = session.get()
def date = new Date()
if(!flowFile) return
flowFile = session.write(flowFile, {inputStream, outputStream ->
SXSSFSheet sheet1 = null;
CSVReader reader = null;
Workbook wb = null;
def nextLine = ''
reader = new CSVReader( inputStream.newReader('UTF-8') );
wb = new SXSSFWorkbook();
sheet1 = (SXSSFSheet) wb.createSheet('Sheet');
def rowNum = 0
while((nextLine = reader.readNext()) != null) {
Row currentRow = sheet1.createRow(rowNum++);
for(int i=0; i < nextLine.length; i++) {
if(NumberUtils.isDigits(nextLine[i])) {
currentRow.createCell(i).setCellValue(Integer.parseInt(nextLine[i]));
} else if (NumberUtils.isNumber(nextLine[i])) {
currentRow.createCell(i).setCellValue(Double.parseDouble(nextLine[i]));
} else {
currentRow.createCell(i).setCellValue(nextLine[i]);
}
}
}
wb.write(outputStream);
reader.close();
} as StreamCallback)
def filename = 'toto.xlsx'
flowFile = session.putAttribute(flowFile, 'filename', filename)
session.transfer(flowFile, REL_SUCCESS)

Nifi - Groovy Json Script define null attributes

I have a Nifi Groovy Script. Assigns fields to nifi attributes but it define null if json values ​​are empty. if json value is null I want to define attribute empty.
null attributes look like this ;
I want to like this ;
this is the script i use ;
import org.apache.commons.io.IOUtils
import java.nio.charset.*
def flowFile = session.get();
if (flowFile == null) {
return;
}
def slurper = new groovy.json.JsonSlurper()
def attrs = [:] as Map<String,String>
session.read(flowFile,
{ inputStream ->
def text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
def obj = slurper.parseText(text)
obj.each {k,v ->
attrs[k] = v.toString()
}
} as InputStreamCallback)
flowFile = session.putAllAttributes(flowFile, attrs)
session.transfer(flowFile, REL_SUCCESS)
flowFile = session.putAllAttributes(flowFile, attrs.collectEntries{k,v->[k,v?:'']})
This is i solved it ;
import org.apache.commons.io.IOUtils
import java.nio.charset.*
def flowFile = session.get();
if (flowFile == null) {
return;
}
def slurper = new groovy.json.JsonSlurper()
def attrs = [:] as Map<String,String>
session.read(flowFile,
{ inputStream ->
def text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
def obj = slurper.parseText(text)
obj.each {k,v ->
if (v.toString()=="null")
{
attrs[k] = ''
} else {
attrs[k] = v.toString()
}
}
} as InputStreamCallback)
flowFile = session.putAllAttributes(flowFile, attrs)
session.transfer(flowFile, REL_SUCCESS)

Excel formula is not updating when SOAP UI writes the data

When I run my SOAP UI project, it works fine and it updates the result i.e pass or fail in the specific column however whatever formula I have written which will take input from pass / fail result is not getting updated.
Can you please let me know any reason?
I am using SOAP UI 5.4
The code is given below
import jxl.*;
import java.io.*;
import java.util.Iterator;
import org.apache.poi.ss.usermodel.*;
import static org.apache.poi.ss.usermodel.CellStyle.*;
import static org.apache.poi.ss.usermodel.IndexedColors.*;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.hssf.usermodel.HSSFFormulaEvaluator;
import groovy.json.*;
import groovy.utils.*;
def soapTestCase = context.testCase
def requestPropertyVariable = soapTestCase.getTestStepByName("requestProperty")
def globalProperties = com.eviware.soapui.model.propertyexpansion.PropertyExpansionUtils.globalProperties
def responseText = context.expand('${responseText}')
def inputFilePath = globalProperties.getPropertyValue("inputFilePath")
def testDataID = globalProperties.getPropertyValue("testDataID")
def testCaseID = globalProperties.getPropertyValue("testCaseID")
int outputRowCount = Integer.parseInt((requestPropertyVariable.getPropertyValue("outputRowCount")))
def responseStatus = testRunner.testCase.getTestStepByName(testCaseID).getTestRequest().response.responseHeaders.find{ it.key == "#status#" }?.value.getAt(0)
int k = Integer.parseInt((requestPropertyVariable.getPropertyValue("outIteration")))
FileInputStream xlwb = new FileInputStream(new File(inputFilePath));
def inptDataWb = new HSSFWorkbook(xlwb);
def inputxlsh = inptDataWb.getSheetAt(2);
def outputxlsh = inptDataWb.getSheetAt(3);
String otestDataID = outputxlsh.getRow(k+1).getCell(0);
String eleName = outputxlsh.getRow(k+1).getCell(1);
DataFormatter formatter = new DataFormatter();
String expectedResult = formatter.formatCellValue(outputxlsh.getRow(k+1).getCell(2));
String mulEleFLg = outputxlsh.getRow(k+1).getCell(4);
//Validate results
if (responseStatus == "HTTP/1.1 200 OK") {
log.info "Response status code check pass"
def groovyUtils = new com.eviware.soapui.support.GroovyUtils( context )
def responseHolder = groovyUtils.getXmlHolder(testRunner.testCase.testSteps[testCaseID].testRequest.response.getContentAsXml())
def tagElement = responseHolder.getNodeValue(eleName) as String //tagElement is a value of a tag
if(mulEleFLg=="Y"){
responseResultArray = tagElement
multiElementValidation(expectedResult, responseResultArray, inptDataWb, outputxlsh, inputFilePath, k)
}else{
if(expectedResult==tagElement.toString().replaceAll('\\[','').replaceAll('\\]','').trim()){
FileOutputStream xlOwb = new FileOutputStream(new File(inputFilePath));
wrtResult = outputxlsh.getRow(k+1).getCell(3);
wrtResult.setCellValue("P");
wrtResult = outputxlsh.getRow(k+1).getCell(5);
wrtResult.setCellValue("");
inptDataWb.write(xlOwb);
xlOwb.close();
}else{
FileOutputStream xlOwb = new FileOutputStream(new File(inputFilePath));
wrtResult = outputxlsh.getRow(k+1).getCell(3);
wrtResult.setCellValue("F");
wrtResult = outputxlsh.getRow(k+1).getCell(5);
wrtResult.setCellValue("Expected Result is " + expectedResult + " but actual result is " + tagElement.toString().replaceAll('\\[','').replaceAll('\\]',''));
log.info "Expected Result is " + expectedResult + " but actual result is " + tagElement.toString().replaceAll('\\[','').replaceAll('\\]','')
inptDataWb.write(xlOwb);
xlOwb.close();
}
}
FormulaEvaluator evaluator = inptDataWb.getCreationHelper().createFormulaEvaluator();
for (Sheet sheet : inptDataWb) {
for (Row r : sheet) {
for (Cell c : r) {
if (c.getCellType() == Cell.CELL_TYPE_FORMULA) {
evaluator.evaluateAll();
inptDataWb.setForceFormulaRecalculation(true);
log.info "Refreshed"
}
}
}
}
}
def multiElementValidation(expectedResult, responseResultArray, inptDataWb, outputxlsh, inputFilePath, k){
int resResArrCnt = responseResultArray.size()
FileOutputStream xlOwb = new FileOutputStream(new File(inputFilePath));
boolean match = false;
if(resResArrCnt==1){
for (item in responseResultArray)
{
for (itemOfItem in item){
log.info itemOfItem
if(expectedResult==itemOfItem.toString().replaceAll('\\[','').replaceAll('\\]','').trim()){
wrtResult = outputxlsh.getRow(k+1).getCell(3);
wrtResult.setCellValue("P");
wrtResult = outputxlsh.getRow(k+1).getCell(5);
wrtResult.setCellValue("");
inptDataWb.write(xlOwb);
match = true;
break;
}
}
}
}else{
for (item in responseResultArray)
{
if(expectedResult==item.toString().replaceAll('\\[','').replaceAll('\\]','').trim()){
wrtResult = outputxlsh.getRow(k+1).getCell(3);
wrtResult.setCellValue("P");
wrtResult = outputxlsh.getRow(k+1).getCell(5);
wrtResult.setCellValue("");
inptDataWb.write(xlOwb);
match = true;
break;
}
}
}
if(!match){
wrtResult = outputxlsh.getRow(k+1).getCell(3);
wrtResult.setCellValue("F");
wrtResult = outputxlsh.getRow(k+1).getCell(5);
wrtResult.setCellValue("Expected Result is " + expectedResult + " but not exist in the array");
log.info "Expected Result is " + expectedResult + " but not exist in the array"
inptDataWb.write(xlOwb);
}
xlOwb.close()
}
def evaluateWorkbook(){
def inputFilePath = globalProperties.getPropertyValue("inputFilePath")
FileInputStream xlwb = new FileInputStream(new File(inputFilePath));
def inptDataWb = new HSSFWorkbook(xlwb);
FormulaEvaluator evaluator = inptDataWb.getCreationHelper().createFormulaEvaluator();
for (Sheet sheet : inptDataWb) {
for (Row r : sheet) {
for (Cell c : r) {
if (c.getCellType() == Cell.CELL_TYPE_FORMULA) {
evaluator.evaluateFormulaCell(c);
log.info "Refreshed the cell"
}
}
}
}
}
I have handle it in excel macro vba under openworkbook. So the workbook formula get updated when the user opens the workbook

Groovy class to extract json returns null

I'm in the process of adding to the Library of reusable functions. I've added a class to extract JSON from the SOAP response but I get a NULL in return. The script below works fine, the issue is when I add this to a class.
import com.eviware.soapui.support.types.StringToStringMap
import groovy.json.JsonSlurper
import groovy.json.*
responseContent = testRunner.testCase.getTestStepByName("TestStepName").getPropertyValue("response");
slurperresponse = new JsonSlurper().parseText(responseContent);
log.info (slurperresponse.items[0].id);
//Write response to property
def addResponseID = slurperresponse.items[0].id
testRunner.testCase.testSuite.setPropertyValue("ScheduleId", addResponseID)
The Class:
import com.eviware.soapui.support.types.StringToStringMap;
import groovy.json.JsonSlurper;
import groovy.json.*;
class GetXmlNode
{
def log
def context
def testRunner
def responseSOAXmlStep
def resultValue
def responseNodePath
def responseContent = "mycontent"
def slurperresponse = "myresponse"
def storeProperty = "id"
// Class constructor with same case as Class name
def GetXmlNode(logIn,contextIn,testRunnerIn)
{
this.log = logIn
this.context = contextIn
this.responseSOAXmlStep = responseSOAXmlStep
this.responseNodePath = responseNodePath
this.testRunner = testRunnerIn
this.storeProperty = storeProperty
}
def execute(responseSOAXmlStep,responseNodePath,storeProperty)
{
responseContent = testRunner.testCase.getTestStepByName(responseSOAXmlStep).getPropertyValue("response");
slurperresponse = new JsonSlurper().parseText(responseContent);
resultValue = (slurperresponse.responseNodePath);
log.info("Node value: " + resultValue );
testRunner.testCase.testSuite.project.setPropertyValue(storeProperty, resultValue );
return responseSOAXmlStep
}
}
context.setProperty( "GetXmlNode", new GetXmlNode( log, context, testRunner) )
The Calling script:
// get a reference to the library TestSuite
library = testRunner.getTestCase().getTestSuite().getProject().getWorkspace().getProjectByName("Library")
library = library.getTestSuiteByName("Library");
// find the module within the library
module = library.testCases["Library"].testSteps["GetXMLNodeValueByJSON"]
// initialise the library; which places an instance of Example in the context
module.run(testRunner, context)
// get the instance of example from the context.
def GetXmlNode = context.GetXmlNode
// run the method, with parameter
log.info "GetXmlNode.execute(responseSOAXmlStep,responseNodePath,storeProperty) = " + GetXmlNode.execute("TestStepName","items[0].id","JsonId")
The Response:
Wed Nov 19 14:00:58 CAT 2014:INFO:Node value: null
Updated Code:
import com.eviware.soapui.support.types.StringToStringMap;
import groovy.json.JsonSlurper;
import groovy.json.*;
class GetXmlNode
{
def log
def context
def testRunner
def responseSOAXmlStep
def resultValue
def responseNodePath = "items[0].id"
def responseContent = ""
def slurperresponse = ""
def storeProperty = ""
// Class constructor with same case as Class name
def GetXmlNode(logIn,contextIn,testRunnerIn)
{
this.log = logIn
this.context = contextIn
this.responseSOAXmlStep = responseSOAXmlStep
this.responseNodePath = responseNodePath
this.testRunner = testRunnerIn
this.storeProperty = storeProperty
}
def execute(responseSOAXmlStep,responseNodePath,storeProperty)
{
def groovyUtils = new com.eviware.soapui.support.GroovyUtils( context );
responseContent = testRunner.testCase.getTestStepByName(responseSOAXmlStep).getProperty("Response").getValue();
def jsonSlurper = new JsonSlurper().parseText(responseContent);
// log.info (slurperresponse.responseNodePath);
// def holder = groovyUtils.getXmlHolder( responseSOAXmlStep+"#ResponseAsXml" );
resultValue = "resultValue " + jsonSlurper[responseNodePath];
log.info("Node value: " + resultValue );
log.info("Response Node Path: " +responseNodePath)
testRunner.testCase.testSuite.project.setPropertyValue(storeProperty, resultValue );
return responseSOAXmlStep
}
}
context.setProperty( "GetXmlNode", new GetXmlNode( log, context, testRunner) )
Response:
Tue Dec 02 10:16:33 CAT 2014:INFO:Node value: resultValue null

Resources