Dynamic object graph navigation in Groovy - groovy

folks!
I want to be able to navigate Groovy object graph dynamically, having the path in string:
def person = new Person("john", new Address("main", new Zipcode("10001", "1234")))
def path = 'address.zip.basic'
I know that I can access a property in map notation, but it's only one level deep:
def path = 'address'
assert person[path] == address
Is there any way to evaluate deeper path?
Thanks!

This can be achieved by overriding the getAt operator and traversing the property graph. The following code uses Groovy Category but inheritance or mixins could also be used.
class ZipCode {
String basic
String segment
ZipCode(basic, segment) {
this.basic = basic
this.segment = segment
}
}
class Address {
String name
ZipCode zip
Address(String name, ZipCode zip) {
this.name = name
this.zip = zip
}
}
class Person {
String name
Address address
Person(String name, Address address) {
this.name = name
this.address = address
}
}
#Category(Object)
class PropertyPath {
static String SEPARATOR = '.'
def getAt(String path) {
if (!path.contains(SEPARATOR)) {
return this."${path}"
}
def firstPropName = path[0..path.indexOf(SEPARATOR) - 1]
def remainingPath = path[path.indexOf(SEPARATOR) + 1 .. -1]
def firstProperty = this."${firstPropName}"
firstProperty[remainingPath]
}
}
def person = new Person('john', new Address('main', new ZipCode('10001', '1234')))
use(PropertyPath) {
assert person['name'] == 'john'
assert person['address.name'] == 'main'
assert person['address.zip.basic'] == '10001'
}
PropertyPath.SEPARATOR = '/'
use(PropertyPath) {
assert person['address/zip/basic'] == '10001'
}

Related

Nested JSON with duplicate keys

I will have to process 10 billion Nested JSON records per day using NiFi (version 1.9). As part of the job, am trying to convert the nested JSON to csv using Groovy script. I referred the below Stack Overflow questions related to the same topic and came up with the below code.
Groovy collect from map and submap
how to convert json into key value pair completely using groovy
But am not sure how to retrieve the value of duplicate keys. Sample json is defined in the variable "json" in the below code. key "Flag1" will be coming in multiple sections (i.e., "OF" & "SF"). I want to get the output as csv.
Below is the output if I execute the below groovy code 2019-10-08 22:33:29.244000,v12,-,36178,0,0/0,10.65.5.56,sf,sf (flag1 key value is replaced by that key column's last occurrence value)
I am not an expert in Groovy. Also please suggest if there is any other better approach, so that I will give a try.
import groovy.json.*
def json = '{"transaction":{"TS":"2019-10-08 22:33:29.244000","CIPG":{"CIP":"10.65.5.56","CP":"0"},"OF":{"Flag1":"of","Flag2":"-"},"SF":{"Flag1":"sf","Flag2":"-"}}'
def jsonReplace = json.replace('{"transaction":{','{"transaction":[{').replace('}}}','}}]}')
def jsonRecord = new JsonSlurper().parseText(jsonReplace)
def columns = ["TS","V","PID","RS","SR","CnID","CIP","Flag1","Flag1"]
def flatten
flatten = { row ->
def flattened = [:]
row.each { k, v ->
if (v instanceof Map) {
flattened << flatten(v)
} else if (v instanceof Collection && v.every {it instanceof Map}) {
v.each { flattened << flatten(it) }
} else {
flattened[k] = v
}
}
flattened
}
print "output: " + jsonRecord.transaction.collect {row -> columns.collect {colName -> flatten(row)[colName]}.join(',')}.join('\n')
Edit: Based on the reply from #cfrick and #stck, I have tried the option and have follow up question below.
#cfrick and #stck- Thanks for your response.
Original source JSON record will have more than 100 columns and I am using "InvokeScriptedProcessor" in NiFi to trigger the Groovy script.
Below is the original Groovy script am using in "InvokeScriptedProcessor" in which I have used Streams(inputstream, outputstream). Is this what you are referring.
Am I doing anything wrong?
import groovy.json.JsonSlurper
class customJSONtoCSV implements Processor {
def REL_SUCCESS = new Relationship.Builder().name("success").description("FlowFiles that were successfully processed").build();
def log
static def flatten(row, prefix="") {
def flattened = new HashMap<String, String>()
row.each { String k, Object v ->
def key = prefix ? prefix + "_" + k : k;
if (v instanceof Map) {
flattened.putAll(flatten(v, k))
} else {
flattened.put(key, v.toString())
}
}
return flattened
}
static def toCSVRow(HashMap row) {
def columns = ["CIPG_CIP","CIPG_CP","CIPG_SLP","CIPG_SLEP","CIPG_CVID","SIPG_SIP","SIPG_SP","SIPG_InP","SIPG_SVID","TG_T","TG_R","TG_C","TG_SDL","DL","I_R","UAP","EDBL","Ca","A","RQM","RSM","FIT","CSR","OF_Flag1","OF_Flag2","OF_Flag3","OF_Flag4","OF_Flag5","OF_Flag6","OF_Flag7","OF_Flag8","OF_Flag9","OF_Flag10","OF_Flag11","OF_Flag12","OF_Flag13","OF_Flag14","OF_Flag15","OF_Flag16","OF_Flag17","OF_Flag18","OF_Flag19","OF_Flag20","OF_Flag21","OF_Flag22","OF_Flag23","SF_Flag1","SF_Flag2","SF_Flag3","SF_Flag4","SF_Flag5","SF_Flag6","SF_Flag7","SF_Flag8","SF_Flag9","SF_Flag10","SF_Flag11","SF_Flag12","SF_Flag13","SF_Flag14","SF_Flag15","SF_Flag16","SF_Flag17","SF_Flag18","SF_Flag19","SF_Flag20","SF_Flag21","SF_Flag22","SF_Flag23","SF_Flag24","GF_Flag1","GF_Flag2","GF_Flag3","GF_Flag4","GF_Flag5","GF_Flag6","GF_Flag7","GF_Flag8","GF_Flag9","GF_Flag10","GF_Flag11","GF_Flag12","GF_Flag13","GF_Flag14","GF_Flag15","GF_Flag16","GF_Flag17","GF_Flag18","GF_Flag19","GF_Flag20","GF_Flag21","GF_Flag22","GF_Flag23","GF_Flag24","GF_Flag25","GF_Flag26","GF_Flag27","GF_Flag28","GF_Flag29","GF_Flag30","GF_Flag31","GF_Flag32","GF_Flag33","GF_Flag34","GF_Flag35","VSL_VSID","VSL_TC","VSL_MTC","VSL_NRTC","VSL_ET","VSL_HRES","VSL_VRES","VSL_FS","VSL_FR","VSL_VSD","VSL_ACB","VSL_ASB","VSL_VPR","VSL_VSST","HRU_HM","HRU_HD","HRU_HP","HRU_HQ","URLF_CID","URLF_CGID","URLF_CR","URLF_RA","URLF_USM","URLF_USP","URLF_MUS","TCPSt_WS","TCPSt_SE","TCPSt_WSFNS","TCPSt_WSF","TCPSt_EM","TCPSt_RSTE","TCPSt_MSS","NS_OPID","NS_ODID","NS_EPID","NS_TrID","NS_VSN","NS_LSUT","NS_STTS","NS_TCPPR","CQA_NL","CQA_CL","CQA_CLC","CQA_SQ","CQA_SQC","TS","V","PID","RS","SR","CnID","A_S","OS","CPr","CVB","CS","HS","SUNR","SUNS","ML","MT","TCPSL","CT","MS","MSH","SID","SuID","UA","DID","UAG","CID","HR","CRG","CP1","CP2","AIDF","UCB","CLID","CLCL","OPTS","PUAG","SSLIL"]
return columns.collect { column ->
return row.containsKey(column) ? row.get(column) : ""
}.join(',')
}
#Override
void initialize(ProcessorInitializationContext context) {
log = context.getLogger()
}
#Override
Set<Relationship> getRelationships() {
return [REL_SUCCESS] as Set
}
#Override
void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException {
try {
def session = sessionFactory.createSession()
def flowFile = session.get()
if (!flowFile) return
flowFile = session.write(flowFile,
{ inputStream, outputStream ->
def bufferedReader = new BufferedReader(new InputStreamReader(inputStream, 'UTF-8'))
def jsonSlurper = new JsonSlurper()
def line
def header = "CIPG_CIP,CIPG_CP,CIPG_SLP,CIPG_SLEP,CIPG_CVID,SIPG_SIP,SIPG_SP,SIPG_InP,SIPG_SVID,TG_T,TG_R,TG_C,TG_SDL,DL,I_R,UAP,EDBL,Ca,A,RQM,RSM,FIT,CSR,OF_Flag1,OF_Flag2,OF_Flag3,OF_Flag4,OF_Flag5,OF_Flag6,OF_Flag7,OF_Flag8,OF_Flag9,OF_Flag10,OF_Flag11,OF_Flag12,OF_Flag13,OF_Flag14,OF_Flag15,OF_Flag16,OF_Flag17,OF_Flag18,OF_Flag19,OF_Flag20,OF_Flag21,OF_Flag22,OF_Flag23,SF_Flag1,SF_Flag2,SF_Flag3,SF_Flag4,SF_Flag5,SF_Flag6,SF_Flag7,SF_Flag8,SF_Flag9,SF_Flag10,SF_Flag11,SF_Flag12,SF_Flag13,SF_Flag14,SF_Flag15,SF_Flag16,SF_Flag17,SF_Flag18,SF_Flag19,SF_Flag20,SF_Flag21,SF_Flag22,SF_Flag23,SF_Flag24,GF_Flag1,GF_Flag2,GF_Flag3,GF_Flag4,GF_Flag5,GF_Flag6,GF_Flag7,GF_Flag8,GF_Flag9,GF_Flag10,GF_Flag11,GF_Flag12,GF_Flag13,GF_Flag14,GF_Flag15,GF_Flag16,GF_Flag17,GF_Flag18,GF_Flag19,GF_Flag20,GF_Flag21,GF_Flag22,GF_Flag23,GF_Flag24,GF_Flag25,GF_Flag26,GF_Flag27,GF_Flag28,GF_Flag29,GF_Flag30,GF_Flag31,GF_Flag32,GF_Flag33,GF_Flag34,GF_Flag35,VSL_VSID,VSL_TC,VSL_MTC,VSL_NRTC,VSL_ET,VSL_HRES,VSL_VRES,VSL_FS,VSL_FR,VSL_VSD,VSL_ACB,VSL_ASB,VSL_VPR,VSL_VSST,HRU_HM,HRU_HD,HRU_HP,HRU_HQ,URLF_CID,URLF_CGID,URLF_CR,URLF_RA,URLF_USM,URLF_USP,URLF_MUS,TCPSt_WS,TCPSt_SE,TCPSt_WSFNS,TCPSt_WSF,TCPSt_EM,TCPSt_RSTE,TCPSt_MSS,NS_OPID,NS_ODID,NS_EPID,NS_TrID,NS_VSN,NS_LSUT,NS_STTS,NS_TCPPR,CQA_NL,CQA_CL,CQA_CLC,CQA_SQ,CQA_SQC,TS,V,PID,RS,SR,CnID,A_S,OS,CPr,CVB,CS,HS,SUNR,SUNS,ML,MT,TCPSL,CT,MS,MSH,SID,SuID,UA,DID,UAG,CID,HR,CRG,CP1,CP2,AIDF,UCB,CLID,CLCL,OPTS,PUAG,SSLIL"
outputStream.write("${header}\n".getBytes('UTF-8'))
while (line = bufferedReader.readLine()) {
def jsonReplace = line.replace('{"transaction":{','{"transaction":[{').replace('}}}','}}]}')
def jsonRecord = new JsonSlurper().parseText(jsonReplace)
def a = jsonRecord.transaction.collect { row ->
return flatten(row)
}.collect { row ->
return toCSVRow(row)
}
outputStream.write("${a}\n".getBytes('UTF-8'))
}
} as StreamCallback)
session.transfer(flowFile, REL_SUCCESS)
session.commit()
}
catch (e) {
throw new ProcessException(e)
}
}
#Override
Collection<ValidationResult> validate(ValidationContext context) { return null }
#Override
PropertyDescriptor getPropertyDescriptor(String name) { return null }
#Override
void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) { }
#Override
List<PropertyDescriptor> getPropertyDescriptors() {
return [] as List
}
#Override
String getIdentifier() { return null }
}
processor = new customJSONtoCSV()
If I should not use "collect" then what else I need to use to create the rows.
In the output flow file, the record output is coming inside []. I tried the below but it is not working. Not sure whether am doing the right thing. I want csv output without []
return toCSVRow(row).toString()
If you know what you want to extract exactly (and given you want to
generate a CSV from it) IMHO you are way better off to just shape the
data in the way you later want to consume it. E.g.
def data = new groovy.json.JsonSlurper().parseText('[{"TS":"2019-10-08 22:33:29.244000","CIPG":{"CIP":"10.65.5.56","CP":"0"},"OF":{"Flag1":"of","Flag2":"-"},"SF":{"Flag1":"sf","Flag2":"-"}}]')
extractors = [
{ it.TS },
{ it.V },
{ it.PID },
{ it.RS },
{ it.SR },
{ it.CIPG.CIP },
{ it.CIPG.CP },
{ it.OF.Flag1 },
{ it.SF.Flag1 },]
def extract(row) {
extractors.collect{ it(row) }
}
println(data.collect{extract it})
// ⇒ [[2019-10-08 22:33:29.244000, null, null, null, null, 10.65.5.56, 0, of, sf]]
As stated in the other answer, due to the sheer amount of data you are trying to
convert::
Make sure to use a library to generate the CSV file from that, or else
you will hit problems with the content, you try to write (e.g. line
breaks or the data containing the separator char).
Don't use collect (it is eager) to create the rows.
The idea is to modify "flatten" method - it should differentiate between same nested keys by providing parent key as a prefix.
I've simplified code a bit:
import groovy.json.*
def json = '{"transaction":{"TS":"2019-10-08 22:33:29.244000","CIPG":{"CIP":"10.65.5.56","CP":"0"},"OF":{"Flag1":"of","Flag2":"-"},"SF":{"Flag1":"sf","Flag2":"-"}}'
def jsonReplace = json.replace('{"transaction":{','{"transaction":[{').replace('}}','}}]')
def jsonRecord = new JsonSlurper().parseText(jsonReplace)
static def flatten(row, prefix="") {
def flattened = new HashMap<String, String>()
row.each { String k, Object v ->
def key = prefix ? prefix + "." + k : k;
if (v instanceof Map) {
flattened.putAll(flatten(v, k))
} else {
flattened.put(key, v.toString())
}
}
return flattened
}
static def toCSVRow(HashMap row) {
def columns = ["TS","V","PID","RS","SR","CnID","CIP","OF.Flag1","SF.Flag1"] // Last 2 keys have changed!
return columns.collect { column ->
return row.containsKey(column) ? row.get(column) : ""
}.join(', ')
}
def a = jsonRecord.transaction.collect { row ->
return flatten(row)
}.collect { row ->
return toCSVRow(row)
}.join('\n')
println a
Output would be:
2019-10-08 22:33:29.244000, , , , , , , of, sf

Groovy: writing dynamically compiled class to disc

In my app I need to compile classes / scripts in runtime.
as a Class:
Class<? extends LoginAdapter> clazz = groovyClassLoader.loadClass name
LoginAdapter la = clazz.newInstance id, logo
or as a Closure:
Closure action = groovyShell.evaluate( script, name ) as Closure
Both ways work like charm.
Now I need to be able to write the compiled classes/scripts to some persistant storage (disc) and later restore them back without compiling from scratch.
How can this be done?
For those who might be interested, this is my programming kata:
class Base {
String answer() { null }
String question() { 'WHAT IS.... ?' }
}
class ClazzSpec extends Specification {
def "test compile"() {
given:
String packageName = 'some.pckg'
String body = """
package $packageName
class SomeClass extends Base {
Closure cl = { a -> println a }
#Override String answer(){ '42' }
String json( m ){ JsonOutput.toJson( m ) }
String longOne( int times ){
times.times{ sleep 100 }
'done'
}
}
"""
CompilerConfiguration cc = new CompilerConfiguration( targetDirectory:new File( './out' ) )
ImportCustomizer imp = new ImportCustomizer()
imp.addStarImports 'groovy.json', Base.package.name
cc.addCompilationCustomizers imp, new ASTTransformationCustomizer( value:1, TimedInterrupt )
new Compiler( cc ).compile packageName + '.SomeClass', body
File base = new File( './out' )
GroovyClassLoader gcl = new GroovyClassLoader()
gcl.addClasspath './out'
Class clazz = gcl.loadClass 'some.pckg.SomeClass'
when:
def inst = clazz.newInstance()
then:
inst.question() == 'WHAT IS.... ?'
inst.answer() == '42'
inst.json( [ q:42 ] ) == '{"q":42}'
inst.longOne( 2 ) == 'done'
when:
inst.longOne 11
then:
thrown TimeoutException
}
}

Simple way to dynamically replace placeholder with value at Groovy strings

I'm trying to find laconic and efficient way to replace placeholders with values at the Groovy Strings. But I can't find convenient solution for 2 cases:
When String with the placeholder and the value are defined at different classes.
When the String is passed as argument to a method, and should be replaced with local's variable value. Here is the illustration of 2 approaches I have tried:
class A {
static def strPlaceHolder = 'token = ${tokenValue}';
static def strRefPlaceHolder = "token = ${->tokenRef}";
}
class B {
def tokenRef = "token reference as field";
void parseGString(GString str) {
println str; //fails here. No property tokenRef for class: A. Though I've expected that "this" is B
}
void parseString(String str) {
def tokenValue = "token value as local variable";
println str; //I know why it doesn't work as required. But how to make something similar
}
}
new B().parseString(A.strPlaceHolder); //token = ${tokenValue}
new B().parseGString(A.strRefPlaceHolder); //fails,
You could replace your GString fields with closures and pass those closures to your methods. e.g.:
class A {
static def strPlaceHolder = { token -> "token = ${token}" }
}
class B {
def tokenRef = "token reference as field";
void parseGString(def closure) {
println closure(tokenRef)
}
void parseString(def closure) {
def tokenValue = "token value as local variable"
println closure(tokenValue)
}
}
new B().parseString(A.strPlaceHolder);
new B().parseGString(A.strPlaceHolder);

Print the specific field from list

Please help to print the assetNumber. Need to search for specific assetname and print its associated assetNumber using list in groovy.
Currently, it does not print value. It struck once search criteria is entered.
class Main
{
public static void main(String[] args)
{
BufferedReader br = new BufferedReader(new InputStreamReader(System.in))
List list = new ArrayList()
Asset asset = new Asset()
def name
def assetNumber
def assigneeName
def assignedDate
def assetType
String userInput = "Yes"
while(userInput.equalsIgnoreCase("Yes"))
{
println "Enter the asset details:"
asset.name = br.readLine()
asset.assetNumber= Integer.parseInt(br.readLine())
asset.assigneeName = br.readLine()
asset.assignedDate = Date.parse('dd/MM/yyyy', br.readLine())
list.add(asset)
println "Do you want to continue(yes/no)"
userInput = br.readLine()
}
println "Enter the asset type:"
assetType = br.readLine()
println "Asserts with type "+assetType+":"
def items = list.findAll{p->p.name == assetType }
items.each { println it.assetNumber }
}
}
class Asset
{
def name
def assetNumber
def assigneeName
def assignedDate
}
You are overwriting the value in your Asset object, then adding the same object to the list every time
If you create a new asset inside the loop (rather than outside it), it should work...
Something like this:
import groovy.transform.*
class Main {
static main(args) {
def console = System.console()
def list = []
while (true) {
def name = console.readLine 'Enter the asset name:'
def number = console.readLine 'Enter the asset number:'
def assignee = console.readLine 'Assignee name:'
def date = console.readLine 'Date (dd/MM/yyyy):'
list << new Asset(
name: name,
assetNumber: Integer.parseInt(number),
assigneeName: assignee,
assignedDate: Date.parse('dd/MM/yyyy', date)
)
if ('Y' != console.readLine('Enter another (y/n):')?.take(1)?.toUpperCase()) {
break
}
}
def type = console.readLine 'Enter the asset type:'
println "Assets with type $type:"
list.findAll { p -> p.name == type }
.each { println it.assetNumber }
}
#Canonical
static class Asset {
def name
def assetNumber
def assigneeName
def assignedDate
}
}

groovy change bean variables using properties

Is there a way in groovy to do something like:
class Person{
def name, surname
}
public void aMethod(anoherBean){
def bean = retrieveMyBean()
p.properties = anoherBean.properties
}
The property properties is final, is there another way to do this shortcut?
properties is a virtual property; you have to call the individual setters. Try this:
def values = [name: 'John', surname: 'Lennon']
for( def entry : values.entries() ) {
p.setProperty( entry.getKey(), entry.getValue() );
}
Or, using MOP:
Object.class.putAllProperties = { values ->
for( def entry : values.entries() ) {
p.setProperty( entry.getKey(), entry.getValue() );
}
}
Person p = new Person();
p.putAllProperties [name: 'John', surname: 'Lennon']
[EDIT] To achieve what you want, you must loop over the properties. This blog post describes how to do that:
def copyProperties(def source, def target){
target.metaClass.properties.each{
if (source.metaClass.hasProperty(source, it.name) && it.name != 'metaClass' && it.name != 'class')
it.setProperty(target, source.metaClass.getProperty(source, it.name))
}
}
If you don't have any special reason then just use named parameters
def p = new Person(name: 'John', surname: 'Lennon')
After question being updated
static copyProperties(from, to) {
from.properties.each { key, value ->
if (to.hasProperty(key) && !(key in ['class', 'metaClass']))
to[key] = value
}
}

Resources