Python3 - recursively replace all keys with periods in a nested dictionary - python-3.x

I am trying to clean up a nested dictionary before inserting it into Mongo. Some of the keys in the dict have periods in them so I need to replace them with underscores. Based on other posts I have seen I have come up with this (not working) code sample:
def get_recursively(search_dict):
new_dict = {}
for key, value in search_dict.items():
if '.' in key or ' ' in key:
new_dict[key.replace('.', '_').replace(' ', '_').lower()] = value
elif isinstance(value, dict):
results = get_recursively(value)
for key2, value2 in results.items():
new_dict[key] = dict(key2, value2)
elif isinstance(value, list):
for item in value:
if isinstance(item, dict):
more_results = get_recursively(item)
for key3, value3 in more_results.items():
new_dict[key] = dict(key3, value3)
else:
new_dict[key] = value
return new_dict
I am trying to make a new dictionary because when I tried to modify the existing dictionary I got an error about the dictionary changing during execution.
The code that is not valid (at least) is:
dict(key2, value2)
That is not valid syntax but hopefully shows my thought process at least.
Any help much appreciated.

If I understood right, is this want you meant?
def change_chars(string, chars, new_char):
new_string = string
for char in chars:
new_string = new_string.replace(char, new_char)
return new_string
def recursively_change_keys(obj, chars, new_char):
if isinstance(obj, list):
return [
recursively_change_keys(o, chars, new_char)
for o in obj
]
elif isinstance(obj, dict):
return {
change_chars(key, chars, new_char): recursively_change_keys(value, chars, new_char)
for key, value in obj.items()
}
return obj
So you just have to call it like recursively_change(search_dict, [ ".", " " ], "_")

Try:
import json
d = {
"some.key": [
{
"key.1": {"a": 1},
"key.2": 2,
"key.3": {"key.4": [3, 4, 5], "key.5": 6},
}
]
}
def transform(d):
if isinstance(d, dict):
return {k.replace(".", "_"): transform(v) for k, v in d.items()}
elif isinstance(d, list):
return [transform(v) for v in d]
else:
return d
# pretty print the dictionary:
print(json.dumps(transform(d), indent=4))
Prints:
{
"some_key": [
{
"key_1": {
"a": 1
},
"key_2": 2,
"key_3": {
"key_4": [
3,
4,
5
],
"key_5": 6
}
}
]
}

Related

Getting error when doing re-mapping dict keys to another dict keys in python

below is my code:
mapping_dict = {"NET_D":
[
("name", "tiN"),
("d_id", "id"),
("m_ip", "ti_ip"),
("model", "cmbM"),
("dc", "cmbL"),
("vendor", "cmbV"),
("cab", "cmbC")
]
}
obj = {"ti_ip": "1.1.1.1", "cmbM": "model-a", "tiN": "device-123", "cmbV": "Systems", "cmbCt": "406", "cmbC": "sc", "id": "199"}
def process_results(item_list, mapping):
results = []
for i in item_list:
item = {}
for m in mapping:
try:
item[m[0]] = i[m[1]]
except KeyError:
item[m[0]] = ""
results.append(item)
return results, len(results)
process_results(obj, mapping_dict["NET_D"])
desired/wanted output:
{"m_ip": "1.1.1.1", "model": "model-a", "name": "device-123", "vendor": "Systems", "cab": "406", "dc": "sc", "d_id": "199"}
error i am getting:
process_results
item[m[0]] = i[m[1]]
TypeError: string indices must be integers
can anyone suggest the right way to achieve desired/wanted output
i am still new to python, apologies for the mistakes/errors or if my code sounds like silly/dumb ;-) to you
You could do this, although technically your mapping_dict is a list of tuples and not a nested dict.
mapping_dict = {"NET_D":
[
("name", "tiN"),
("d_id", "id"),
("m_ip", "ti_ip"),
("model", "cmbM"),
("dc", "cmbL"),
("vendor", "cmbV"),
("cab", "cmbC")
]
}
obj = {"ti_ip": "1.1.1.1", "cmbM": "model-a", "tiN": "device-123", "cmbV": "Systems", "cmbCt": "406", "cmbC": "sc", "id": "199"}
def process_results(item_list, mapping):
return {i[0]:v for k,v in item_list.items() for i in mapping if k == i[1]}
which will give
{'m_ip': '1.1.1.1', 'model': 'model-a', 'name': 'device-123', 'vendor': 'Systems', 'cab': 'sc', 'd_id': '199'}```
This is called dict comprehension and creates a new dictionary.
It is basically doing the equivalent of
def process_results(item_list, mapping):
res = {}
for k,v in item_list.items():
for i in mapping:
if k == i[1]:
res[i[0]] = v
return res
Iterating for each value of the obj dict, then iterate through the mapping list of tuples and if the value is the same as index[1] of the tuple then create a new key:value in the new dict.

How to add currency sign to all integers in a list of nested dictionaries

Given a list of nested dictionaries how can I add the currency locale to all the integer values using the locale module. My current solution works however I could not figure out how to make it work with nested dictionaries nor does it feel pythonic.
Example input
[
{
'name':'Tom',
'total_salary': 70000,
'salary': {
'base': 65000,
'bonus': 5000
}
},
{
'name':'Andrew',
'total_salary': 50000,
'salary': {
'base': 45000,
'bonus': 5000
}
}
]
Wanted output
[
{
'name':'Tom',
'total_salary': '$70000',
'salary': {
'base': '$65000',
'bonus': '$5000'
}
},
{
'name':'Andrew',
'total_salary': '$50000',
'salary': {
'base': '$45000',
'bonus': '$5000'
}
}
]
current solution
import locale
locale.setlocale( locale.LC_ALL, 'en_CA.UTF-8' )
def add_currency_locale(_list):
new_list = []
for d in _list:
for k,v in list(d.items()):
try:
v = float(v)
new_value = locale.currency(v, grouping=True)
d[k] = new_value
except:
pass
new_list.append(d)
return new_list
Because you have the line locale.setlocale( locale.LC_ALL, 'en_CA.UTF-8' ), I think you don't want to have the local currency symbol, but want it to always be '$'. If so, here is my solution, otherwise you can easily replace the line where I set new_value. I'm using recursion to correctly handle cases when you have nested lists or dictionaries (the code you've provided seems to not work for those cases, but according to the example input and output you need this. If you don't, remove the part with instance checking and replace the line except ValueError: with except (ValueError, TypeError):). Pay attention to the notes I left in the comments
# Note: variable names with one leading underscore are "private" according to python code style.
# Use trailing underscore instead
def add_currency_locale(list_):
new_list = []
for d in list_:
# Note: no need to convert `d.items()` to list: you can iterate over the original
# object, and the conversion takes time
for k, v in d.items():
if isinstance(v, dict):
# Because `add_currency_locale` only works with arrays, make an array of one
# dictionary and then only use the first (and only) element of the returned list
d[k] = add_currency_locale([v])[0]
elif isinstance(v, list):
d[k] = add_currency_locale(v)
elif isinstance(v, (int, float)):
d[k] = f'${v}'
else:
d[k] = v
new_list.append(d)
return new_list
You can use this code to iterate through all the data elements and get the $ symbol assigned to each value.
def add_curr(dict_list):
new_lst = []
for dic in dict_list:
for k1, v1 in dic.items():
if not isinstance(v1, dict):
if isinstance(v1, (int, float)):
dic[k1] = '${}'.format(v1)
else:
dic[k1] = v1
else:
for k2,v2 in v1.items():
isinstance(v2, (int, float)):
dic[k1][k2] = '${}'.format(v2)
else:
dic[k1][k2] = v2
new_lst.append(dic)
return new_lst
add_curr(mylist)
You could decide to use the conversion part as a separate function and call it each time with a key and value
if isinstance(v1, (int, float)):
dic[k1] = '${}'.format(v1)
else:
dic[k1] = v1
This will give you the following dict:
[{'name': 'Tom',
'total_salary': '$70000',
'salary':
{'base': '$65000',
'bonus': '$5000'}},
{'name': 'Andrew',
'total_salary': '$50000',
'salary':
{'base': '$45000',
'bonus': '$5000'}}]
If you want to use the locale module:
lst = [
{
'name':'Tom',
'total_salary': 70000,
'salary': {
'base': 65000,
'bonus': 5000
}
},
{
'name':'Andrew',
'total_salary': 50000,
'salary': {
'base': 45000,
'bonus': 5000
}
}
]
import locale
locale.setlocale( locale.LC_ALL, 'en_CA.UTF-8' )
def cnvrt(dct):
for k, v in dct.items():
if isinstance(v, dict):
dct[k] = cnvrt(v)
elif isinstance(v, int) or isinstance(v, float):
dct[k] = locale.currency(v, grouping=True)
return dct
print([cnvrt(i) for i in lst])
However, that won't give you your expected output which is just a $ prepended to the value.
For that you can use.
def cnvrt(dct):
for k, v in dct.items():
if isinstance(v, dict):
dct[k] = cnvrt(v)
elif isinstance(v, int) or isinstance(v, float):
dct[k] = f'${v}'
return dct
print([cnvrt(i) for i in lst])
This works by recursively calling cnvrt if the value is a nested dict, otherwise, if it's an int or float it prepends a $. Since it's operating on the expectation of a dict object, you can use it in a nice list comprehension.
Finally, if you really want to you can handle lists with your function but IMO at this point it is doing too much.
def cnvrt(obj):
if isinstance(obj, list):
obj = [cnvrt(i) for i in obj]
elif isinstance(obj, dict):
for k, v in obj.items():
if isinstance(v, dict) or isinstance(v, list):
obj[k] = cnvrt(v)
elif isinstance(v, int) or isinstance(v, float):
obj[k] = f'${v}'
return obj
print(cnvrt(lst))
Although, it works with inner lists and dicts as well, There is a lot going on which makes it hard to follow.

python recursion avoid result as global variable

res_dict = {}
def get_value(co , passed_dict ):
for k, v in passed_dict.items():
if isinstance(v, dict):
get_value(co, v)
else:
res_dict[k] = v
print("from else",res_dict)
return res_dict
def easy():
inner_dict = {
"test1" : {"test1_in" : "abc"},
"test2" : {"test1_in" : "xyz"}
}
dict1 = {}
count = 0
val_from_function= {}
key_list = ['key1','key2']
for key in key_list:
count = count + 1
val_from_function = get_value(count ,inner_dict)
print("before assign" ,dict1 )
dict1[key] = val_from_function
print("after assign" , dict1)
# dict1['key1'] = {'test1' : "abc"}
# dict1['key2'] = {'test1' : "xyz"}
print(dict1)
easy()
receiving output : {'key1': {'test1_in': 'xyz'}, 'key2': {'test1_in': 'xyz'}}
expected o/p : {'key1': {'test1_in': 'abc'}, 'key2': {'test1_in': 'xyz'}}
I understand the value of dict1 is updated with the last value as res_dict declared as global
variable.
I can solve it by appending the inner key value with outer key and then storing in dictionary.
I might solve it using ordered dictionary.
taking keys from list as outer key value (key1, key2 ..,key3000) is unknown.
Looking for suggestions on how to make this code better with expected o/p.
Have 3k key-pair values, same as sample pattern with more nested k,v & storing o/p as cache, so performance is not a very big issue here.
# This modifies passed_dict
def get_value(passed_dict, recur_dict_new={}, isInitial=True):
for k, v in passed_dict.items():
if isInitial:
recur_dict_new = {}
if isinstance(v, dict):
temp = get_value(v, recur_dict_new, isInitial=False)
passed_dict[k]= temp
else:
recur_dict_new[k]=v
return recur_dict_new
def easy():
inner_dict = {
"test1" : {"test1_in" : "abc"},
"test2" : {"test1_in" : "xyz"}
}
key_list = ['key1','key2']
for key in key_list:
get_value(inner_dict)
# dict1['key1'] = {'test1' : "abc"}
# dict1['key2'] = {'test1' : "xyz"}
print(inner_dict)
easy()
Thanks for looking, I have solved with one of the ways as mentioned above,
tested with 15k records with 3 more levels of nested JSON, the performance was okay(4ms)
O/p : {'test1': {'test1_in': 'abc'}, 'test2': {'test1_in': 'xyz'}}

Find the intersection of dict of dicts based on the rules in python3.x

I have two dictionaries as given below and want to find the intersection of dictionaries based on some logic.
dict1= {"1":{"score1": 1.099, "score2":0.45},
"2": {"score2": 0.099, "score3":1.45},
"3": {"score2": 10, "score3":10.45}}
dict2= {"1":{"score6": 1.099, "score2":0.45},
"2": {"score2": 10, "score3":10.45},
"4": {"score5": 8, "score8":15}}
I want to create the dictionary based on the given two dictionaries based on the below rules:
1.union of the two dicitonaries based on the outer key
if outer key is common in both the dictionaries then in the nested key-value pair show only the common key with highest value across both the dictionaries.
result_dict = {"1":{"score2":0.45},
"3": {"score2": 10, "score3":10.45},
"2": {"score2": 10, "score3":10.45},
"4": {"score5": 8, "score8":15}}```
First off, thanks for providing concrete examples of what your inputs are like and what you'd like the output to look like.
There may well be more efficient ways of doing this, but since there's no mention of any constraints on performance, my first instinct was to turn to Python's set operations to make things a little simpler:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
dict1 = {
"1": {
"score1": 1.099,
"score2": 0.45
},
"2": {
"score2": 0.099,
"score3": 1.45
},
"3": {
"score2": 10,
"score3": 10.45
}
}
dict2 = {
"1": {
"score6": 1.099,
"score2": 0.45
},
"2": {
"score2": 10,
"score3": 10.45
},
"4": {
"score5": 8,
"score8": 15
}
}
result_dict = {
"1": {
"score2": 0.45
},
"3": {
"score2": 10,
"score3": 10.45
},
"2": {
"score2": 10,
"score3": 10.45
},
"4": {
"score5": 8,
"score8": 15
}
}
def weird_union(d1, d2):
"""Applies the logic in OP's question
Args:
d1 (dict): dict with one level of nested dicts as values
d2 (dict): dict with one level of nested dicts as values
Returns: dict
"""
result = {}
k1, k2 = set(d1.keys()), set(d2.keys())
# no collisions-- easy case
for k in k1.symmetric_difference(k2):
result[k] = d1[k] if k in d1 else d2[k]
# key appears in both dicts
for k in k1.intersection(k2):
_k1, _k2 = set(d1[k].keys()), set(d2[k].keys())
result[k] = {
key: max([d1[k][key], d2[k][key]])
for key in _k1.intersection(_k2)
}
return result
test = weird_union(dict1, dict2)
assert result_dict == test
print('Test passed.')
The basic idea is to treat the disjoint and the intersection cases separately. Hope this helps.
Update in response to comment:
In the future, please provide this sort of context up front; an operation on two dictionaries is rather different than an operation on an arbitrary number of inputs.
Here's one way to do it:
def invert_dicts(*dicts):
""" Takes multiple dicts and returns a dict mapping
key to dict index. E.g.,
invert_dicts(
{'a': 1, 'b': 2},
{'a': 3, 'c': 4}
)
returns
{'a': [0, 1], 'b': [0], 'c': [1]}
"""
key_map = {}
for i, d in enumerate(dicts):
for k in d.keys():
key_map.setdefault(k, []).append(i)
return key_map
def weird_n_union(*dicts):
"""Applies the logic in OP's question to an arbitrary number of inputs
>>> weird_n_union(d1, d2, ..., dn)
Args:
*dicts (dict): dictionaries w/one level of nested dicts as values
Returns: dict
"""
result = {}
# dict mapping key to list of dict index in `dicts` containing key
key_map = invert_dicts(*dicts)
for k in key_map:
# no outer key collision
if len(key_map[k]) == 1:
result[k] = dicts[key_map[k][0]][k]
# outer key collision
else:
# unclear what should happen in the case where:
# - there is an outer key collision
# - there are no shared sub-keys
#
# this implementation assumes that in that case, the value for k is {}
result.setdefault(k, {})
sub_dicts = tuple(dicts[i][k] for i in key_map[k])
# map keys in `sub_dicts` to indices for `dicts` containing key
sub_key_map = invert_dicts(*sub_dicts)
# contains elements of (k, v), where k appears in > 1 sub-dicts
shared_keys_only = filter(lambda kv: len(kv[1]) > 1,
sub_key_map.items())
# update result with the max value for each shared key
for kv in shared_keys_only:
max_ = max(((kv[0], sub_dicts[i][kv[0]]) for i in kv[1]),
key=lambda x: x[1])
result[k].update({max_[0]: max_[1]})
return result
Tried to annotate to make it a bit clear how things work. Hopefully this works for your use case.

Print nested dictionary in python3

How can I print a nested python dictionary in a specific format?
So, my dictionary is looks like this:
dictionary = {'Doc1':{word1: 3, word2: 1}, 'Doc2':{word1: 1, word2: 14, word3: 3}, 'Doc3':{word1: 2}}
I tried the following way:
for x, y in dictionary.items():
print(x,":", y)
But it will printL`
Doc1: {word1:3, word2: 1}
Doc2: {word1:1, word2:14, word3:3}
Doc3: {word1:2}
How to get rid of the bracket and print the plain information?
I want to print on the following format:
Doc1: word1:3; word2:1
Doc2: word1:1; word2:14; word3: 3
Doc3: word1:2;
:
in your case 'y' is a dict, so if you want to print it differently you can override the repr (representation of the object) or dict.
alternatively you can use some recursion here
def print_d(dd):
if type(dd) != dict:
return str(dd)
else:
res = []
for x,y in dd.items():
res.append(''.join((str(x),':',print_d(y))))
return '; '.join(res)
if __name__=='__main__':
dictionary = {'Doc1':{'word1': 3, 'word2': 1}, 'Doc2':{'word1': 1, 'word2': 14, 'word3': 3}, 'Doc3':{'word1': 2}}
for x, y in dictionary.items():
print(x,": ", print_d(y))
Aside from the fact that your original dictionary declaration is not valid python unless each word is a defined variable, this seems to work:
import json
print(json.dumps(dictionary).replace("{","").replace(',','').replace("}","\n").replace('"',''))
Result:
Doc1: word1: 3 word2: 1
Doc2: word1: 1 word2: 14 word3: 3
Doc3: word1: 2

Resources