I have a Nifi Groovy Script. Assigns fields to nifi attributes but it define null if json values are empty. if json value is null I want to define attribute empty.
null attributes look like this ;
I want to like this ;
this is the script i use ;
import org.apache.commons.io.IOUtils
import java.nio.charset.*
def flowFile = session.get();
if (flowFile == null) {
return;
}
def slurper = new groovy.json.JsonSlurper()
def attrs = [:] as Map<String,String>
session.read(flowFile,
{ inputStream ->
def text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
def obj = slurper.parseText(text)
obj.each {k,v ->
attrs[k] = v.toString()
}
} as InputStreamCallback)
flowFile = session.putAllAttributes(flowFile, attrs)
session.transfer(flowFile, REL_SUCCESS)
flowFile = session.putAllAttributes(flowFile, attrs.collectEntries{k,v->[k,v?:'']})
This is i solved it ;
import org.apache.commons.io.IOUtils
import java.nio.charset.*
def flowFile = session.get();
if (flowFile == null) {
return;
}
def slurper = new groovy.json.JsonSlurper()
def attrs = [:] as Map<String,String>
session.read(flowFile,
{ inputStream ->
def text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
def obj = slurper.parseText(text)
obj.each {k,v ->
if (v.toString()=="null")
{
attrs[k] = ''
} else {
attrs[k] = v.toString()
}
}
} as InputStreamCallback)
flowFile = session.putAllAttributes(flowFile, attrs)
session.transfer(flowFile, REL_SUCCESS)
Related
I am using Apache Nifi to extract data SQL query into XLS file.
I use ExecuteQuery to extract data into Avro, then to CSV using CSV RecordWriter
Now I should to have my data in an XLS file, to do, I am using this Groovy script but it's not working:
// import org.apache.commons.io.IOUtils
import java.nio.charset.*
// import java.text.SimpleDateFormat
import java.io.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.hssf.usermodel.*
import org.apache.poi.xssf.usermodel.*
import org.apache.poi.ss.util.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.hssf.extractor.*
def flowFile = session.get()
if(!flowFile) return
flowFile = session.write(flowFile, {inputStream, outputStream ->
try {
inputStream.writeTo(outputStream)
// i tried also outputStream.write(inputStream)
//i tried also to retrieve the excel file with:
//Workbook wb = WorkbookFactory.create(inputStream)
//and write with : outputStream.write(wb)
}
catch(e) {
log.error("Error during processing", e)
session.transfer(flowFile, REL_FAILURE)
}
} as StreamCallback)
session.transfer(flowFile, REL_SUCCESS)
I have this error:
Edit:
I changed my script and had an other error:
#Grapes(#Grab(group='org.apache.poi', module='poi-ooxml', version='3.9'))
import com.opencsv.CSVReader
#Grapes(#Grab(group='com.opencsv', module='opencsv', version='4.2'))
import org.apache.poi.ss.usermodel.*
import org.apache.poi.xssf.streaming.*
import org.apache.poi.hssf.usermodel.*
import org.apache.poi.xssf.usermodel.*
import org.apache.poi.ss.util.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.hssf.extractor.*
import java.nio.charset.*
import java.io.*
import org.apache.commons.io.IOUtils
def flowFile = session.get()
def date = new Date()
if(!flowFile) return
flowFile = session.write(flowFile, {inputStream, outputStream ->
SXSSFSheet sheet1 = null;
CSVReader reader = null;
Workbook wb = null;
String generatedXlsFilePath = "/home/";
FileOutputStream fileOutputStream = null;
def filename = flowFile.getAttribute('filename')
def path = flowFile.getAttribute('path')
def nextLine = ''
reader = new CSVReader(new FileReader(path+filename), ',');
//Workbook wb = WorkbookFactory.create(inputStream,);
//Sheet sheet1 = wb.createSheet("Feuille");
wb = new SXSSFWorkbook(inputStream);
sheet1 = (SXSSFSheet) wb.createSheet('Sheet');
def rowNum = 0;
while((nextLine = reader.readNext()) != null) {
Row currentRow = sheet1.createRow(rowNum++);
for(int i=0; i < nextLine.length; i++) {
if(NumberUtils.isDigits(nextLine[i])) {
currentRow.createCell(i).setCellValue(Integer.parseInt(nextLine[i]));
} else if (NumberUtils.isNumber(nextLine[i])) {
currentRow.createCell(i).setCellValue(Double.parseDouble(nextLine[i]));
} else {
currentRow.createCell(i).setCellValue(nextLine[i]);
}
}
}
//fileOutputStream = new FileOutputStream(generatedXlsFilePath.trim());
//wb.write(fileOutputStream);
generatedXlsFilePath = generatedXlsFilePath + 'SAISIE_MAGASING.XLS'
outputStream = new FileOutputStream(generatedXlsFilePath.trim());
wb.write(outputStream);
wb.close();
//fileOutputStream.close();
outputStream.close();
reader.close();
//outputStream.close();
inputStream.close();
} as StreamCallback)
flowFile = session.putAttribute(flowFile, 'filename', filename)
The new error:
Thanks to #dagget
This is the worked script:
#Grapes(#Grab(group='org.apache.poi', module='poi-ooxml', version='3.9'))
import com.opencsv.CSVReader
#Grapes(#Grab(group='com.opencsv', module='opencsv', version='4.2'))
//#Grapes(#Grab(group='org.apache.commons', module='lang', version='3.12.0'))
#Grapes(#Grab(group='commons-lang', module='commons-lang', version='2.4'))
import org.apache.commons.lang.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.xssf.streaming.*
import org.apache.poi.hssf.usermodel.*
import org.apache.poi.xssf.usermodel.*
import org.apache.poi.ss.util.*
import org.apache.poi.ss.usermodel.*
import org.apache.poi.hssf.extractor.*
import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import java.nio.charset.*
import java.io.*
import org.apache.commons.io.IOUtils
def flowFile = session.get()
def date = new Date()
if(!flowFile) return
flowFile = session.write(flowFile, {inputStream, outputStream ->
SXSSFSheet sheet1 = null;
CSVReader reader = null;
Workbook wb = null;
def nextLine = ''
reader = new CSVReader( inputStream.newReader('UTF-8') );
wb = new SXSSFWorkbook();
sheet1 = (SXSSFSheet) wb.createSheet('Sheet');
def rowNum = 0
while((nextLine = reader.readNext()) != null) {
Row currentRow = sheet1.createRow(rowNum++);
for(int i=0; i < nextLine.length; i++) {
if(NumberUtils.isDigits(nextLine[i])) {
currentRow.createCell(i).setCellValue(Integer.parseInt(nextLine[i]));
} else if (NumberUtils.isNumber(nextLine[i])) {
currentRow.createCell(i).setCellValue(Double.parseDouble(nextLine[i]));
} else {
currentRow.createCell(i).setCellValue(nextLine[i]);
}
}
}
wb.write(outputStream);
reader.close();
} as StreamCallback)
def filename = 'toto.xlsx'
flowFile = session.putAttribute(flowFile, 'filename', filename)
session.transfer(flowFile, REL_SUCCESS)
I'm pretty new to Groovy an I want to import a class from another groovy script in the main groovy script. I saw already this post. However due to environment restrictions I can't us functions like GroovyClassLoader or or import any libs/frameworks. The only possibility I have is to use the evaluate method.
I tried this:
main script (The evaluate does not throw any error)
File csvFile = new File("./Csvreader.groovy");
evaluate(csvFile);
Csvreader script
class CSVReader{
def csvFile = new File('./test.csv');
def separatorChar = ";"
def values = [];
def headers = [];
void setSeperator(String s){
if(s != ""){
separatorChar = s;
}
}
void readCsv(){
csvFile.eachLine{ line, number ->
if(number == 1){
def head = line.split(separatorChar);
for (entry in head) {
headers.add(entry);
}
} else{
def value = line.split(separatorChar);
def map =[:];
def i = 0;
for (entry in value) {
map.put(headers[i], entry);
i++;
}
values.add(map);
}
}
}
def getValues(){
return values;
}
def getHeaders(){
return headers;
}
def getSize(){
return values.size();
}
def getLine(def keyParam, def value){
for(int i = 0; i < values.size(); i++){
def line = values[i];
if(values[i][keyParam] == value){
return values[i];
}
}
}
}
However I would need to pass parameters with the evaluate call (and I would move the methods outside the class) or I need to create an instance of the class.
Unfortunately I don't really know how I can do that, does anyone have a solution for that?
you could try to put at the end of your Csvreader.groovy file the expression return CSVReader.class that will return you a compiled class.
Then in your caller script you could create an instance of this class and use it as any other class instance.
Csvreader.groovy
class CSVReader{
def separatorChar = ";"
void setSeparator(String s){
if(s != ""){
separatorChar = s;
}
}
}
return CSVReader.class
caller script
File csvFile = new File("./Csvreader.groovy")
def CSVReaderClass = evaluate(csvFile)
def cvsReader = CSVReaderClass.newInstance()
cvsReader.setSeparator("Z")
println cvsReader.separatorChar
I need covert xml files in Nifi 1.5. I try execute this script in ExcecuteScript processor:
import groovy.xml.*
import javax.xml.transform.TransformerFactory
import javax.xml.transform.Transformer
import javax.xml.transform.dom.DOMSource
import javax.xml.transform.stream.StreamResult
import org.apache.commons.io.IOUtils
import java.nio.charset.StandardCharsets
import org.apache.nifi.processor.io.StreamCallback
import java.io.StringWriter
flowFile = session.get()
if(!flowFile) return
session.write(flowFile,
{
inputStream, outputStream ->
def xml = IOUtils.toString(inputStream, 'UTF-8')
def reader = new StringReader(xml)
def doc = DOMBuilder.parse(reader)
def records = doc.documentElement
TransformerFactory tFactory = TransformerFactory.newInstance();
StringWriter writer = new StringWriter();
Transformer transformer = tFactory.newTransformer();
DOMSource source = new DOMSource(records);
StreamResult result = new StreamResult(writer);
String strString = writer.toString()
outputStream.write(strString.getBytes(StandardCharsets.UTF_8))
} as StreamCallback
)
session.transfer(flowFile, REL_SUCCESS)
And Output FlowFile is empty.
In local this part working:
def xml = "<xml>...</xml>"
def reader = new StringReader(xml)
def doc = DOMBuilder.parse(reader)
def records = doc.documentElement
TransformerFactory tFactory = TransformerFactory.newInstance();
StringWriter writer = new StringWriter();
Transfor mer transformer = tFactory.newTransformer();
DOMSource source = new DOMSource(records);
StreamResult result = new StreamResult(writer);
String strString = writer.toString()
Why output file is empty?
I missed the line transformer.transform(source, result)
full script for Nifi:
import groovy.xml.*
import javax.xml.transform.TransformerFactory
import javax.xml.transform.Transformer
import javax.xml.transform.dom.DOMSource
import javax.xml.transform.stream.StreamResult
import org.apache.commons.io.IOUtils
import java.nio.charset.StandardCharsets
import org.apache.nifi.processor.io.StreamCallback
import java.io.StringWriter
flowFile = session.get()
if(!flowFile) return
session.write(flowFile,
{
inputStream, outputStream ->
def xml = IOUtils.toString(inputStream, 'UTF-8')
def reader = new StringReader(xml)
def doc = DOMBuilder.parse(reader)
def records = doc.documentElement
TransformerFactory tFactory = TransformerFactory.newInstance();
StringWriter writer = new StringWriter();
Transformer transformer = tFactory.newTransformer();
DOMSource source = new DOMSource(records);
StreamResult result = new StreamResult(writer);
transformer.transform(source, result)
String strString = writer.toString()
outputStream.write(strString.getBytes(StandardCharsets.UTF_8))
} as StreamCallback
)
session.transfer(flowFile, REL_SUCCESS)
I try to save all the attachments from a SOAP response. I use following Groovy script.
def testStep = testRunner.testCase.getTestStepByName("SubmitFile")
def response = testStep.testRequest.response
assert null != response, "response is null"
def outFile = new FileOutputStream(new File(System.getProperty('java.io.tmpdir')+'/test.zip'))
for(i=0; i<3; i++){
def ins = response.responseAttachments[0].inputStream
if (ins) {
com.eviware.soapui.support.Tools.writeAll(outFile, ins)
}
}
ins.close()
outFile.close()
I get following error message:
No such property : responseAttachments for class
Use following script in assertion (Script assertion) in Soap UI.
def response = messageExchange.response
assert null != response, "response is empty"
def outFile
def inputStream
if(messageExchange.responseAttachments.size() >0){
inputStream = messageExchange.responseAttachments[0].inputStream
if (inputStream) {
outFile = new FileOutputStream(new File('/Users/sys/Documents/test/bank_response.txt'))
com.eviware.soapui.support.Tools.writeAll(outFile, inputStream)
}
}else{
log.error 'No attachments found!!!'
}
if(inputStream) inputStream.close()
if(outFile) outFile.close()
responseAttachments is a property on MessageExchange. You're using Response, so you'd need attachments instead. See the API docs for more details.
def testCaseName = 'Messages-IP-Mail'
def testStepName = '001_1ConventMailToPDF'
//context.testCase.testSuite.testCases[testCaseName].testSteps[testStepName].testRequest.responseContent
def projectDir = context.expand('${projectDir}');
log.info "Current dir:" + projectDir
def response = context.testCase.testSuite.testCases[testCaseName].testSteps[testStepName].testRequest.response.getAttachments()
def fileName = projectDir + '/pdf.pdf'
def outFile = new FileOutputStream(new File(fileName))
testRunner.testCase.testSteps["fileName"].setPropertyValue("fileName", fileName)
def ins = response[0].inputStream
if (ins) {
com.eviware.soapui.support.Tools.writeAll(outFile, ins)
}
ins.close()
outFile.close()
I'm in the process of adding to the Library of reusable functions. I've added a class to extract JSON from the SOAP response but I get a NULL in return. The script below works fine, the issue is when I add this to a class.
import com.eviware.soapui.support.types.StringToStringMap
import groovy.json.JsonSlurper
import groovy.json.*
responseContent = testRunner.testCase.getTestStepByName("TestStepName").getPropertyValue("response");
slurperresponse = new JsonSlurper().parseText(responseContent);
log.info (slurperresponse.items[0].id);
//Write response to property
def addResponseID = slurperresponse.items[0].id
testRunner.testCase.testSuite.setPropertyValue("ScheduleId", addResponseID)
The Class:
import com.eviware.soapui.support.types.StringToStringMap;
import groovy.json.JsonSlurper;
import groovy.json.*;
class GetXmlNode
{
def log
def context
def testRunner
def responseSOAXmlStep
def resultValue
def responseNodePath
def responseContent = "mycontent"
def slurperresponse = "myresponse"
def storeProperty = "id"
// Class constructor with same case as Class name
def GetXmlNode(logIn,contextIn,testRunnerIn)
{
this.log = logIn
this.context = contextIn
this.responseSOAXmlStep = responseSOAXmlStep
this.responseNodePath = responseNodePath
this.testRunner = testRunnerIn
this.storeProperty = storeProperty
}
def execute(responseSOAXmlStep,responseNodePath,storeProperty)
{
responseContent = testRunner.testCase.getTestStepByName(responseSOAXmlStep).getPropertyValue("response");
slurperresponse = new JsonSlurper().parseText(responseContent);
resultValue = (slurperresponse.responseNodePath);
log.info("Node value: " + resultValue );
testRunner.testCase.testSuite.project.setPropertyValue(storeProperty, resultValue );
return responseSOAXmlStep
}
}
context.setProperty( "GetXmlNode", new GetXmlNode( log, context, testRunner) )
The Calling script:
// get a reference to the library TestSuite
library = testRunner.getTestCase().getTestSuite().getProject().getWorkspace().getProjectByName("Library")
library = library.getTestSuiteByName("Library");
// find the module within the library
module = library.testCases["Library"].testSteps["GetXMLNodeValueByJSON"]
// initialise the library; which places an instance of Example in the context
module.run(testRunner, context)
// get the instance of example from the context.
def GetXmlNode = context.GetXmlNode
// run the method, with parameter
log.info "GetXmlNode.execute(responseSOAXmlStep,responseNodePath,storeProperty) = " + GetXmlNode.execute("TestStepName","items[0].id","JsonId")
The Response:
Wed Nov 19 14:00:58 CAT 2014:INFO:Node value: null
Updated Code:
import com.eviware.soapui.support.types.StringToStringMap;
import groovy.json.JsonSlurper;
import groovy.json.*;
class GetXmlNode
{
def log
def context
def testRunner
def responseSOAXmlStep
def resultValue
def responseNodePath = "items[0].id"
def responseContent = ""
def slurperresponse = ""
def storeProperty = ""
// Class constructor with same case as Class name
def GetXmlNode(logIn,contextIn,testRunnerIn)
{
this.log = logIn
this.context = contextIn
this.responseSOAXmlStep = responseSOAXmlStep
this.responseNodePath = responseNodePath
this.testRunner = testRunnerIn
this.storeProperty = storeProperty
}
def execute(responseSOAXmlStep,responseNodePath,storeProperty)
{
def groovyUtils = new com.eviware.soapui.support.GroovyUtils( context );
responseContent = testRunner.testCase.getTestStepByName(responseSOAXmlStep).getProperty("Response").getValue();
def jsonSlurper = new JsonSlurper().parseText(responseContent);
// log.info (slurperresponse.responseNodePath);
// def holder = groovyUtils.getXmlHolder( responseSOAXmlStep+"#ResponseAsXml" );
resultValue = "resultValue " + jsonSlurper[responseNodePath];
log.info("Node value: " + resultValue );
log.info("Response Node Path: " +responseNodePath)
testRunner.testCase.testSuite.project.setPropertyValue(storeProperty, resultValue );
return responseSOAXmlStep
}
}
context.setProperty( "GetXmlNode", new GetXmlNode( log, context, testRunner) )
Response:
Tue Dec 02 10:16:33 CAT 2014:INFO:Node value: resultValue null