Convert Nested Json to a single Key Value Pair - python-3.x

I have a list of Keys :
Server
Date
Severity
process
message
If there is a key "events" in my Dictionary, I need to replace the key with "Message" and the value of "indicator-type" will become the value of the message
List1=[
{
"server": "10.10.0.123",
"date": 1561994754,
"severity": "INFO",
"process": "webapp",
"message": "server started."
},
{
"server": "10.10.0.202",
"date": 1561994757000,
"source": "jvm-x994a",
"events": [{
"indicator-type": "memory-low"
}]
},
{
"server": "10.10.0.202",
"date": 1561994773000,
"source": "jvm-x994a",
"events": [{
"indicator-type": "memory-low"
}
]
}
]
Expected output:
List1=[
{
"server": "10.10.0.123",
"date": 1561994754,
"severity": "INFO",
"process": "webapp",
"message": "server started."
},
{
"server": "10.10.0.202",
"date": 1561994757000,
"source": "jvm-x994a",
"message": "memory-low"
},
{
"server": "10.10.0.202",
"date": 1561994773000,
"source": "jvm-x994a",
"message": "memory-low"
}
]
}
]
So Far I have Tried :
In the below code I am trying to check for the "events" key and replacing it with the "message" key, but when I am printing it is not replacing they key-value
output = []
d={}
for x in a:
d['server']=x['server']
if (str(x['date']).isdigit() == True):
d['date'] =2
# datetime.datetime.fromtimestamp(x['date'] / 1000.0).strftime('%Y-%m-%d %H:%M:%S.%f')
if ("events" in x):
for i in (x['events']):
if (i['indicator-level'] == 3):
d['severity'] = "INFO"
if (i['indicator-level'] == 7):
d['severity'] = "WARN"
if (i['indicator-type'] != None):
d['message'] = i['indicator-type']

Clean out the dictionary first before moving on to the next step.
def handle_events_key(dict_):
"""
Adds a new 'message' key to 'dict_' based on 'events' key.
Returns a new dictionary without the 'events' key.
"""
if 'events' in dict_:
dict_['message'] = dict_['events'][0]['indicator-type']
return {key: value for key, value in dict_.items() if key != 'events'}
def events_key_to_message(list_):
return [handle_events_key(dict_) for dict_ in list_]
# Usage
# List1 = ...
# Cleaning out the dictionary.
ListUpdated = events_key_to_message(List1)

Try this. I have modified the events handling parts.
output = []
for item in List1:
d = {}
d['server'] = item['server']
if (str(item['date']).isdigit() == True):
d['date'] =2
# datetime.datetime.fromtimestamp(x['date'] / 1000.0).strftime('%Y-%m-%d %H:%M:%S.%f')
# get will give you None and not raise an exception if there is no events key.
if item.get('events') is not None:
for event in item['events']:
if event.get('indicator-level') == 3: d['severity'] = "INFO"
if event.get('indicator-level') == 7: d['severity'] = "WARN"
if event.get('indicator-type') is not None: d['message'] = event['indicator-type']
output.append(d)
print(output)
# [{'server': '10.10.0.123', 'date': 2}, {'server': '10.10.0.202', 'date': 2, 'message': 'memory-low'}, {'server': '10.10.0.202', 'date': 2, 'message': 'memory-low'}]

Related

I am prepared permit EIP712, but tx "Reverted", function selector 'Reverted 0x08c379a

I'm trying to send a permit for a transaction, but I get a function call mismatch return if I call through the smartcontract token function. If I call via encodeABI and add the same arguments, I don't see the message in the transaction. I use the following code
def build_permit(owner, spender, value, deadline, web3):
data = {
"types": {
"EIP712Domain": [
{"name": "name", "type": "string"},
{"name": "version", "type": "string"},
{"name": "chainId", "type": "uint256"},
{"name": "verifyingContract", "type": "address"},
],
"Permit": [
{"name": "owner", "type": "address"},
{"name": "spender", "type": "address"},
{"name": "value", "type": "uint256"},
{"name": "nonce", "type": "uint256"},
{"name": "deadline", "type": "uint256"},
],
},
"domain": {
"name": "TestToken",
"version": "1",
"chainId": 4,
"verifyingContract": address_token,
},
"primaryType": "Permit",
"message": {
"owner": owner,
"spender": spender,
"value": value,
"nonce": web3.eth.getTransactionCount(my_account.address),
"deadline": deadline,
},
}
return encode_structured_data(data)
def test_permit():
signer = my_account
owner = my_account.address
holder = signer.address
address_token = SC.USDT("RINKEBY")
spender = SC.ROUTER('RINKEBY')
deadline = int(time.time()) + 3600
#print(deadline)
amount = web3.toWei(1, 'ether')
usdt = token_contract(address_token,web3n(RINKEBY))
permit = build_permit(owner, spender, amount, deadline,web3n(RINKEBY))
signed = signer.sign_message(permit)
tx = usdt.functions.permit(holder, spender, 1000000000000000000, int(time.time()) + 3600, signed.v, Web3.toBytes(signed.r), Web3.toBytes(signed.s)).call()
Why does the beginning of the data start with a different function and how do I get the correct signature?
../PycharmProjects/1/venv/lib/python3.9/site-packages/web3/contract.py:957: in call
return call_contract_function(
../PycharmProjects/1/venv/lib/python3.9/site-packages/web3/contract.py:1501: in call_contract_function
return_data = web3.eth.call(
../PycharmProjects/1/venv/lib/python3.9/site-packages/web3/module.py:57: in caller
result = w3.manager.request_blocking(method_str,
../PycharmProjects/1/venv/lib/python3.9/site-packages/web3/manager.py:198: in request_blocking
return self.formatted_response(response,
../PycharmProjects/1/venv/lib/python3.9/site-packages/web3/manager.py:170: in formatted_response
apply_error_formatters(error_formatters, response)
../PycharmProjects/1/venv/lib/python3.9/site-packages/web3/manager.py:70: in apply_error_formatters
formatted_resp = pipe(response, error_formatters)
cytoolz/functoolz.pyx:667: in cytoolz.functoolz.pipe
???
cytoolz/functoolz.pyx:642: in cytoolz.functoolz.c_pipe
???
response = {'error': {'code': 3, 'data': '0x08c379a000000000000000000000000000000000000000000000000000000000000000200000000000000...207369676e61747572650000', 'message': 'execution reverted: ERC20Permit: invalid signature'}, 'id': 1, 'jsonrpc': '2.0'}
def raise_solidity_error_on_revert(response: RPCResponse) -> RPCResponse:
"""
Reverts contain a `data` attribute with the following layout:
"Reverted "
Function selector for Error(string): 08c379a (4 bytes)
Data offset: 32 (32 bytes)
String length (32 bytes)
Reason string (padded, use string length from above to get meaningful part)
See also https://solidity.readthedocs.io/en/v0.6.3/control-structures.html#revert
"""
if not isinstance(response['error'], dict):
raise ValueError('Error expected to be a dict')
data = response['error'].get('data', '')
# Ganache case:
if isinstance(data, dict) and response['error'].get('message'):
raise ContractLogicError(f'execution reverted: {response["error"]["message"]}')
# Parity/OpenEthereum case:
if data.startswith('Reverted '):
# "Reverted", function selector and offset are always the same for revert errors
prefix = 'Reverted 0x08c379a00000000000000000000000000000000000000000000000000000000000000020' # noqa: 501
if not data.startswith(prefix):
raise ContractLogicError('execution reverted')
reason_length = int(data[len(prefix):len(prefix) + 64], 16)
reason = data[len(prefix) + 64:len(prefix) + 64 + reason_length * 2]
raise ContractLogicError(f'execution reverted: {bytes.fromhex(reason).decode("utf8")}')
# Geth case:
if 'message' in response['error'] and response['error'].get('code', '') == 3:
> raise ContractLogicError(response['error']['message'])
E web3.exceptions.ContractLogicError: execution reverted: ERC20Permit: invalid signature
../PycharmProjects/1/venv/lib/python3.9/site-packages/web3/_utils/method_formatters.py:576: ContractLogicError

Groovy: Compare a response with a json file while ignoring certain keys

I have a response from a REST API with ~40k lines and I need to verify if the values in it match those in an already existing local file. The response does have a certain element which is dynamic and is based on the date and time the response has been received.
I need to compare the response with the local json file and make sure the values match, while ignoring that specific date&time element that comes through.
This is a snippet example of the response/json file:
[
{
"code" : "SJM",
"valuesCount" : [
{
"code" : "SJM",
"description" : "LAST_NAME FIRST_NAME-SJM (LMM/null)",
"period" : "November 2020",
"printedOn" : "31/01/2022 09:39:39",
"name" : "null",
"logo" : { },
"standardId" : true,
"dailyValues" : [
{
"index" : "odd",
"day" : "01",
"description" : "",
"time" : "23:59",
"cms" : "",
"hcv" : "",
"nm" : "",
"hcp" : "",
"hcz" : "",
"synDc" : "",
"jiJnm" : "",
"c1" : "",
"c2" : "",
"proHvs" : "",
"dcd" : "",
"dcs" : "",
"d1" : "",
"d2" : "",
"nbdays" : ""
},
]
}
}
]
]
The file contains hundreds of code entries, each with their own valuesCount maps, in which the the printedOn element appears and that's what I'm trying to ignore.
I've been trying to do this in Groovy and the only thing I've managed to find in the groovy docs is the removeAll() method thinking I could perhaps remove that key:value entirely but I don't think I'm using it right.
import groovy.json.*
def groovyUtils = new com.eviware.soapui.support.GroovyUtils(context)
def projectDir = groovyUtils.projectPath
def response = new JsonSlurper().parseText(context.expand( '${Request#Response}' ))
File jsonFile = new File(projectDir, "/Project/Resources/jsonFile.json")
def defaultValues = new JsonSlurper().parse(jsonFile)
var1 = response.removeAll{"printedOn"}
var2 = defaultValues.removeAll{"printedOn"}
if (var1 == var2) {
log.info "The request matches the default values."
} else {
throw new Exception("The request does not match the default values.");
}
This just returns a true statement for both cases.
Could anyone point me in the right direction?
Much appreciated.
You need to traverse into the hierarchy of lists/maps to alter the inner map and remove the printedOn key/value pair.
Also your json is broken with missing/extra brackets.
The following code:
import groovy.json.*
def data1 = '''\
[
{
"code": "SJM",
"valuesCount": [
{
"code": "SJM",
"description": "LAST_NAME FIRST_NAME-SJM (LMM/null)",
"period": "November 2020",
"printedOn": "31/01/2022 09:39:39",
"name": "null",
"logo": {},
"standardId": true,
"dailyValues": [
{
"index": "odd",
"day": "01",
"description": "",
"time": "23:59",
"cms": "",
"hcv": "",
"nm": "",
"hcp": "",
"hcz": "",
"synDc": "",
"jiJnm": "",
"c1": "",
"c2": "",
"proHvs": "",
"dcd": "",
"dcs": "",
"d1": "",
"d2": "",
"nbdays": ""
}
]
}
]
}
]'''
def data2 = '''\
[
{
"code": "SJM",
"valuesCount": [
{
"code": "SJM",
"description": "LAST_NAME FIRST_NAME-SJM (LMM/null)",
"period": "November 2020",
"printedOn": "25/11/2021 09:39:39",
"name": "null",
"logo": {},
"standardId": true,
"dailyValues": [
{
"index": "odd",
"day": "01",
"description": "",
"time": "23:59",
"cms": "",
"hcv": "",
"nm": "",
"hcp": "",
"hcz": "",
"synDc": "",
"jiJnm": "",
"c1": "",
"c2": "",
"proHvs": "",
"dcd": "",
"dcs": "",
"d1": "",
"d2": "",
"nbdays": ""
}
]
}
]
}
]'''
def parser = new JsonSlurper()
def json1 = parser.parseText(data1)
def json2 = parser.parseText(data2)
def clean1 = removeNoise(json1)
def clean2 = removeNoise(json2)
def out1 = JsonOutput.toJson(clean1)
def out2 = JsonOutput.toJson(clean2)
assert out1 == out2
if(out1 == out2) println "the two documents are identical"
else println "the two documents differ"
def removeNoise(json) {
json.collect { codeMap ->
codeMap.valuesCount = codeMap.valuesCount.collect { valuesMap ->
valuesMap.remove('printedOn')
valuesMap
}
codeMap
}
}
will produce the following output when executed:
─➤ groovy solution.groovy
the two documents are identical
─➤
The json in the above code has been corrected to be parseable. I manually changed the string dates so that data1 and data2 differ on the printedOn property.
edit - it seems that you can actually just do:
assert clean1 == clean2
// or
if (clean1 == clean2) ...
for the comparison. Cuts out the extra serialization step to json but also leaves me feeling a tad uncomfortable with trusting groovy to do the right thing when comparing a nested data structure like this.
I would probably still go with the serialization.

How to convert a dictionary according to a json scheme, Python3

I have a json scheme, which specifies the format of a dictionary in Python 3.
INPUT_SCHEME = {
"type": "object",
"properties": {
"a1": {
"type": "object",
"properties": {
"a1_1": {"type": ["string", "null"]},
"a1_2": {"type": ["number", "null"]},
},
"additionalProperties": False,
"minProperties": 2,
},
"a2": {
"type": "array",
"items": {"type": ["number", "null"]},
},
"a3": {
"type": ["number", "null"],
},
"a4": {
"type": "object",
"properties": {
"a4_1": {"type": ["string", "null"]},
"a4_2": {
"type": "object",
"properties": {
"a4_2_1": {"type": ["string", "null"]},
"a4_2_2": {"type": ["number", "null"]},
},
"additionalProperties": False,
"minProperties": 2,
},
},
"additionalProperties": False,
"minProperties": 2,
},
"a5": {
"type": "array",
"items": {
"type": "object",
"properties": {
"a5_1": {"type": ["string", "null"]},
"a5_2": {"type": ["number", "null"]},
},
"additionalProperties": False,
"minProperties": 2,
},
},
},
"additionalProperties": False,
"minProperties": 5,
}
And I want to write a function which can convert an arbitrary input dictionary to the format defined by the INPUT_SCHEME.
The rules are:
if the input dict misses a filed, then fill the filed with None or empty list in the output dict.
if the input dict has a key that is not defined in the INPUT_SCHEME, then remove it in the output dict.
For example, suppose I have a_input, where only 'a1' is correct. 'a2', 'a3', and 'a4' are missing. Each element in 'a5' misses one property. And 'a6' is an un-defined field.
The function I want to write should convert a_input to a_output. And you can use jsonschema.validate to check.
a_input = {
'a1': {'a1_1': 'apple', 'a1_2': 20.5},
'a5': [{'a5_1': 'pear'}, {'a5_2': 18.5}],
'a6': [1, 2, 3, 4],
}
a_output = {
'a1': {'a1_1': 'apple', 'a1_2': 20.5},
'a2': [],
'a3': None,
'a4': {
'a4_1': None,
'a4_2': {
'a4_2_1': None,
'a4_2_2': None,
}
},
'a5': [
{
'a5_1': 'pear',
'a5_2': None,
},
{
'a5_1': None,
'a5_2': 18.5,
}
]
}
jsonschema.validate(a_output, schema=INPUT_SCHEME)
I tried to write the function, but could not make it. Mainly because there are too many if-else check plus the nested structure, and I got lost. Could you please help me?
Thanks.
def my_func(a_from):
a_to = dict()
for key_1 in INPUT_SCHEME['properties'].keys():
if key_1 not in a_from:
a_to[key_1] = None # This is incorrect, since the structure of a_to[key_1] depends on INPUT_SCHEME.
continue
layer_1 = INPUT_SCHEME['properties'][key_1]
if 'properties' in layer_1: # like a1, a4
for key_2 in layer_1['properties'].keys():
layer_2 = layer_1['properties'][key_2]
...
# but it can be a nest of layers. Like a4, there are 3 layers. In real case, it can have more layers.
elif 'items' in layer_1:
if 'properties' in layer_1['items']: # like a5
...
else: # like a2
...
else: # like 3
...
return a_to
A recursive algorithm suits this.
I divided it into 2 different functions as removing undefined properties and filling non-existent ones from the schema are 2 different tasks. You can merge them into one if you wish.
For filling nonexistent properties, I just create arrays, objects and Nones, and then recurse inwards.
For removing the undefined properties, I compare the schema keys and remove unmatched keys, again, recursing inwards.
You may see comments and type checks in code:
def fill_nonexistent_properties(input_dictionary, schema):
"""
Fill missing properties in input_dictionary according to the schema.
"""
properties = schema['properties']
missing_properties = set(properties).difference(input_dictionary)
# Fill all missing properties.
for key in missing_properties:
value = properties[key]
if value['type'] == 'array':
input_dictionary[key] = []
elif value['type'] == 'object':
input_dictionary[key] = {}
else:
input_dictionary[key] = None
# Recurse inside all properties.
for key, value in properties.items():
# If it's an array of objects, recurse inside each item.
if value['type'] == 'array' and value['items']['type'] == 'object':
object_list = input_dictionary[key]
if not isinstance(object_list, list):
raise ValueError(
f"Invalid JSON object: {key} is not a list.")
for item in object_list:
if not isinstance(item, dict):
raise ValueError(
f"Invalid JSON object: {key} is not a list of objects.")
fill_nonexistent_properties(item, value['items'])
# If it's an object, recurse inside it.
elif value['type'] == 'object':
obj = input_dictionary[key]
if not isinstance(obj, dict):
raise ValueError(
f"Invalid JSON object: {key} is not a dictionary.")
fill_nonexistent_properties(obj, value)
def remove_undefined_properties(input_dictionary, schema):
"""
Remove properties in input_dictionary that are not defined in the schema.
"""
properties = schema['properties']
undefined_properties = set(input_dictionary).difference(properties)
# Remove all undefined properties.
for key in undefined_properties:
del input_dictionary[key]
# Recurse inside all existing sproperties.
for key, value in input_dictionary.items():
property_shcema = properties[key]
# If it's an array of objects, recurse inside each item.
if isinstance(value, list):
if not property_shcema['type'] == 'array':
raise ValueError(
f"Invalid JSON object: {key} is not a list.")
# We're only dealing with objects inside arrays.
if not property_shcema['items']['type'] == 'object':
continue
for item in value:
# Make sure each item is an object.
if not isinstance(item, dict):
raise ValueError(
f"Invalid JSON object: {key} is not a list of objects.")
remove_undefined_properties(item, property_shcema['items'])
# If it's an object, recurse inside it.
elif isinstance(value, dict):
# Make sure the object is supposed to be an object.
if not property_shcema['type'] == 'object':
raise ValueError(
f"Invalid JSON object: {key} is not an object.")
remove_undefined_properties(value, property_shcema)
import pprint
pprint.pprint(a_input)
fill_nonexistent_properties(a_input, INPUT_SCHEME)
remove_undefined_properties(a_input, INPUT_SCHEME)
print("-"*10, "OUTPUT", "-"*10)
pprint.pprint(a_input)
Output:
{'a1': {'a1_1': 'apple', 'a1_2': 20.5},
'a5': [{'a5_1': 'pear'}, {'a5_2': 18.5}],
'a6': [1, 2, 3, 4]}
---------- OUTPUT ----------
{'a1': {'a1_1': 'apple', 'a1_2': 20.5},
'a2': [],
'a3': None,
'a4': {'a4_1': None, 'a4_2': {'a4_2_1': None, 'a4_2_2': None}},
'a5': [{'a5_1': 'pear', 'a5_2': None}, {'a5_1': None, 'a5_2': 18.5}]}

How can I return all nested objects using python?

I wrote an Elastic query which will check the condition (status="APPROVED") and Gets all approved_by objects.
This is my index (portfolio):
{
"settings": {},
"mappings": {
"portfolio": {
"properties": {
"status": {
"type": "keyword",
"normalizer": "lcase_ascii_normalizer"
},
"archived_at": {
"type": "date"
},
"approved_by": {
"id": "text",
"name":"text"
}
}
}
}
}
Currently I have 60 objects whose status are approved , so when i run the query it will show 60 objects,but in my case i am getting only one object(I debugged the code, total 60 objects are coming as expected, but still returning only single object), please help guys.
My query:
profiles = client.search(index='portfolio', doc_type='portfolio',
scroll='10m', size=1000,
body={
"query": {"match": {"status": "APPROVED"}}
})
sid = profiles['_scroll_id']
scroll_size = len(profiles['hits']['hits'])
while scroll_size > 0:
for info in profiles['hits']['hits']:
item = info['_source']
approved_by_obj = item.get('approved_by')
if approved_by_obj:
return (jsonify({"approved_by": approved_by_obj}))
Expected o/p format:
{
"approved_by": {
"id": "system",
"name": "system"
}
}
You're getting only one result because you're returning from your loop, effectively breaking out of it completely.
So, instead of returning from it, append the found approved_by_object to a list of your choice and then return that 60-member list:
profiles = client.search(index='portfolio', doc_type='portfolio',
scroll='10m', size=1000,
body={
"query": {"match": {"status": "APPROVED"}}
})
sid = profiles['_scroll_id']
scroll_size = len(profiles['hits']['hits'])
approved_hits_sources = [] # <-- add this
while scroll_size > 0:
for info in profiles['hits']['hits']:
item = info['_source']
approved_by_obj = item.get('approved_by')
if approved_by_obj:
approved_hits_sources.append({"approved_by": approved_by_obj}) # <--- append and not return
return jsonify({"approved_hits_sources": approved_hits_sources})

How to find object in python list by attribute

If I have a list created like this:
list = [
{
"id": 1,
"name": "something 1"
},
{
"id": 2,
"name": "something 2"
}
]
and I want to grab information from one of the list items, using the id from the url like this:
#route.app('item/<int:id>')
def item(id):
name = ???
return render_template('index.html', name=name)
how would I get the name of the object in the list with the specific id from the url?
A simple for loop will do your work:
#route.app('item/<int:id>')
def item(id):
some_list = [
{
"id": 1,
"name": "something 1"
},
{
"id": 2,
"name": "something 2"
}
]
name = ""
for obj in some_list:
if obj['id'] == id:
name = obj['name']
return render_template('index.html', name=name)

Resources