I need help with a function to flatten a nested dictionary in the following format:
dict_test = {
"subdoc": {
"a": 1,
"b": 2,
"c": 3,
"array": [
{"name": "elmo"},
{"name": "oscar"}
]
}
}
I tried the function below, but is doesn´t flatten the nested keys inside the array element.
def flatten_dict(dd, separator='.', prefix=''):
return { prefix + separator + k if prefix else k : v
for kk, vv in dd.items()
for k, v in flatten_dict(vv, separator, kk).items()
} if isinstance(dd, dict) else { prefix : dd }
My current output is:
{'subdoc.a': 1,
'subdoc.b': 2,
'subdoc.c': 3,
'subdoc.array': [{'name': 'elmo'}, {'name': 'oscar'}]}
But actually I´m looking something like:
{
"subdoc.a": 1,
"subdoc.b": 2,
"subdoc.c": 3,
"subdoc.array.0.name": "elmo",
"subdoc.array.1.name": "oscar"
}
dict_test = {
"subdoc": {
"a": 1,
"b": 2,
"c": 3,
"array": [
{"name": "elmo"},
{"name": "oscar"}
]
}
}
def flatten(d):
agg = {}
def _flatten(d, prev_key=''):
if isinstance(d, list):
for i, item in enumerate(d):
new_k = '%s.%s' % (prev_key, i) if prev_key else i
_flatten(item, prev_key=new_k)
elif isinstance(d, dict):
for k, v in d.items():
new_k = '%s.%s' % (prev_key, k) if prev_key else k
_flatten(v, prev_key=new_k)
else:
agg[prev_key] = d
_flatten(d)
return agg
print(flatten(dict_test))
# gives {'subdoc.a': 1, 'subdoc.b': 2, 'subdoc.c': 3, 'subdoc.array.0.name': 'elmo', 'subdoc.array.1.name': 'oscar'}
Note: only tested for your case
Related
For example, I have two dictionaries having the same keys:
a = {"a": 1, "b": 2, "c":4.5, "d":[1,2], "e":"string", "f":{"f1":0.0, "f2":1.5}}
b = {"a": 10, "b": 20, "c":3.5, "d":[0,2,4], "e":"q", "f":{"f1":1.0, "f2":0.0}}
and I want to compare the types. My code is something like this:
if type(a["a"]) == type(b["a"]) and type(a["b"]) == type(b["b"]) and type(a["c"]) == type(b["c"]) and type(a["d"]) == type(b["d"]) and type(a["e"]) == type(b["e"]) and type(a["f"]) == type(b["f"]) and type(a["f"]["f1"]) == type(b["f"]["f1"]) and type(a["f"]["f2"]) == type(b["f"]["f2"]):
first_type = type(b["d"][0])
if all( (type(x) is first_type) for x in a["d"] )
#do something
pass
Is there a better way to do it?
You can make a list of the common keys between the dicts:
common_keys = a.keys() & b.keys()
and then iterate over them to check the types:
for k in common_keys:
if type(a[k]) == type(b[k]):
print("Yes, same type! " + k, a[k], b[k])
else:
print("Nope! " + k, a[k], b[k])
and if you wanted to go deeper, check if any of the items are dicts, rinse an repeat
for k in common_keys:
if type(a[k]) == type(b[k]):
print("Yes, same type! " + k, type(a[k]), type(b[k]))
if isinstance(a[k], dict):
ck = a[k].keys() & b[k].keys()
for key in ck:
if type(a[k][key]) == type(b[k][key]):
print("Yes, same type! " + key, type(a[k][key]), type(b[k][key]))
else:
print("Nope!")
else:
print("Nope! " + k, type(a[k]), type(b[k]))
You can use a for loop to iterate through the dicts:
same_types = True
for key in a.keys():
if type(a[key]) != type(b[key]):
same_types = False
break
# if the value is a dict, check nested value types
if type(a[key]) == dict:
for nest_key in a[key].keys():
if type(a[key][nest_key]) != type(b[key][nest_key]):
same_types = False
break
# if the value is a list, check all list elements
# I just simply concat two lists together, you can also refer to
# https://stackoverflow.com/q/35554208/19322223
elif type(a[key]) == list:
first_type = a[key][0]
for elem in a[key] + b[key]:
if type(elem) != first_type:
same_types = False
break
if not same_types:
break
if same_types:
# do something
With the following helper function:
def get_types(obj, items=None):
"""Function that recursively traverses 'obj' and returns
a list of all values and nested values types
"""
if not items:
items = []
if isinstance(obj, dict):
for value in obj.values():
if not isinstance(value, (dict, list, set, tuple)):
items.append(value)
else:
get_types(value, items)
elif isinstance(obj, (list, set, tuple)):
for value in obj:
get_types(value, items)
else:
items.append(obj)
return [type(x) for x in items]
You can compare two dictionaries' values types however deeply nested these are, like this:
if get_types(a) == get_types(b):
print("Each a and b values are of same types")
Since, in your example, a misses one value for d key ([1, 2]) compared to the other dict ([0, 2, 4]), nothing will be printed.
Let's take another example where both dictionaries have the same shape this time, but one value of different type (f2):
a = {"a": 1, "b": [[1, 2], [3, [4]]], "c": {"c1": 0.0, "c2": {"x": "9"}}}
b = {"d": 7, "e": [[2, 1], [5, [7]]], "f": {"f1": 8.9, "f2": {"y": 9}}}
if get_types(a) == get_types(b):
print("Each a and b values are of same types")
Then again, nothing will be printed.
But if you replace 9 by "9" in b["f2"]:
a = {"a": 1, "b": [[1, 2], [3, [4]]], "c": {"c1": 0.0, "c2": {"x": "9"}}}
b = {"d": 7, "e": [[2, 1], [5, [7]]], "f": {"f1": 8.9, "f2": {"y": "9"}}}
if get_types(a) == get_types(b):
print("Each a and b values are of same types")
# Output
# Each a and b values are of same types
This question already has answers here:
Accessing nested keys in Python
(5 answers)
Closed 2 years ago.
Is there a way to return a list of all dictionary keys in the provided input (at all levels)?
The keys should be ordered by level, for example:
{
"A": 1,
"B": [{
"B1": 1,
"B2": 1
}, {
"B3": 1,
"B4": 1
}],
"C": {
"C1": 1,
"C2": 1
}
}
I have tried using dict.keys() and dict.items() but did not get the desired output.
The output should be like this ["A", "B", "C", "B1", "B2", "B3", "B4", "C1", "C2"].
Any help is appreciated.
I would do a recursive function:
d = {
"A": 1,
"B": [{
"B1": 1,
"B2": 1
}, {
"B3": 1,
"B4": 1
}],
"C": {
"C1": 1,
"C2": 1
}
}
def flatten(d, lst=None):
if not lst:
lst = []
if isinstance(d, list):
for item in d:
lst = flatten(item, lst)
elif isinstance(d, dict):
for k in d.keys():
lst.append(k)
for v in d.values():
lst = flatten(v, lst)
return lst
print(flatten(d)) # Output: ['A', 'B', 'C', 'B1', 'B2', 'B3', 'B4', 'C1', 'C2']
EDIT: Above code assumes you're using Python 3.7+ where dict are ordered by default. If you're running an older version, you can use collection.OrderedDict instead while initializing d to have the same behavior.
I think i found it
#In order to convert nested list to single list
import more_itertools
a={"A": 1,
"B": [{
"B1": 1,
"B2": 1
}, {
"B3": 1,
"B4": 1
}],
"C": {
"C1": 1,
"C2": 1
}
}
dic_key=[]
for i in a:
dic_key.append(i)
for i in a:
if type(a[i]) is list :
dic_key+=list(map(lambda x:list(x.keys()),a[i]))
elif type(a[i]) is dict:
dic_key+=list(map(lambda x: x,a[i]))
print(list(more_itertools.collapse(dic_key)))
i have a nested python dictionary where i am trying to remove all keys except those mentioned. I found a solution here where we can remove keys specified, what i am trying to do is reverse of it, i want to remove all keys except those mentioned. So i made a slight change in the function, but i am getting an empty dict.
from collections import MutableMapping
def remove_fields(d, list_of_keys_to_remove):
if not isinstance(d, (dict, list)):
return d
if isinstance(d, list):
return [v for v in (remove_fields(v, list_of_keys_to_remove) for v in d) if v]
return {k: v for k, v in ((k, remove_fields(v, list_of_keys_to_remove)) for k, v in d.items()) if
k in list_of_keys_to_remove}
Adding sample input
EDIT: let suppose i want to keep RENEWAL_TYPE
{
'G_1': [
{
'ACCOUNTING_RULE_ID': '1',
'PAYMENT_TERM_ID': '4',
'RENEWAL_TYPE': 'RENEW'
},
{
'ACCOUNTING_RULE_ID': '2',
'PAYMENT_TERM_ID': '4',
'RENEWAL_TYPE': 'RENEW'
},
{
'ACCOUNTING_RULE_ID': '3',
'PAYMENT_TERM_ID': '4',
'RENEWAL_TYPE': 'DO_NOT_RENEW'
},
{
'ACCOUNTING_RULE_ID': '4',
'PAYMENT_TERM_ID': '4',
'RENEWAL_TYPE': 'RENEW'
}
]
}
You're looking for the intersection between the set of keys in the dict and the set of values in the list. i.e. all keys that exist in both sets.
Try this:
from collections import MutableMapping
d = { 'A': 1, 'B': 2, 'C': 3, 'D': 4 }
list_of_keys_to_keep = ['B', 'C', 'F']
def remove_fields(d, list_of_keys_to_keep):
return {key: value for key, value in d.items() if key in list_of_keys_to_keep}
print(remove_fields(d, list_of_keys_to_keep)) # -> {'B': 2, 'C': 3}
Edit: Updated name of list_of_keys_to_remove to list_of_keys_to_keep since this seems to be what it actually represents.
Edit2: Updated after Askers update. The following works for the sample provided:
from collections import MutableMapping
d = {
'G_1': [
{
'ACCOUNTING_RULE_ID': '1',
'PAYMENT_TERM_ID': '4',
'RENEWAL_TYPE': 'RENEW'
},
{
'ACCOUNTING_RULE_ID': '2',
'PAYMENT_TERM_ID': '4',
'RENEWAL_TYPE': 'RENEW'
},
{
'ACCOUNTING_RULE_ID': '3',
'PAYMENT_TERM_ID': '4',
'RENEWAL_TYPE': 'DO_NOT_RENEW'
},
{
'ACCOUNTING_RULE_ID': '4',
'PAYMENT_TERM_ID': '4',
'RENEWAL_TYPE': 'RENEW'
}
]
}
list_of_keys_to_keep = ['RENEWAL_TYPE']
def filter_dict(di, keys):
if not isinstance(di, (dict, list)):
return di
if isinstance(di, list):
return [filter_dict(value, keys) for value in di]
return {key: filter_dict(value, keys) for key, value in di.items() if key in keys or isinstance(value, (dict, list))}
print(filter_dict(d, list_of_keys_to_keep))
Output:
{'G_1': [{'RENEWAL_TYPE': 'RENEW'}, {'RENEWAL_TYPE': 'RENEW'}, {'RENEWAL_TYPE': 'DO_NOT_RENEW'}, {'RENEWAL_TYPE': 'RENEW'}]}
try this one:
dict = {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5}
keys_to_preserve = ["e", "b", "c"]
dict_final={k: v for k,v in dict.items() if k in keys_to_preserve}
def remove_fields(d, keys_to_preserve):
if isinstance(d, list):
return [remove_fields(v, keys_to_preserve) for v in d]
elif isinstance(d, dict):
return dict([(k, remove_fields(v, keys_to_preserve)) for k, v in d.items() if k in keys_to_preserve])
else:
return d
I have a very long dictionary
mydict = {
"6574": [],
"3234": [1],
"7014": [],
"0355": [3],
"1144": [2],
# …
}
I need to get all keys that have a non-empty list.
mytruedict = {}
for k, v in mydict.items():
if v:
mytruedict[k]=v
I was wondering if there is a one-line approach to it.
Using dict
Ex:
mydict = {
"6574": [],
"3234": [1],
"7014": [],
"0355": [3],
"1144": [2]
}
print( dict((k, v) for k, v in mydict.items() if v) )
#or
print( {k: v for k, v in mydict.items() if v } ) #dict comprehension
Output:
{'3234': [1], '1144': [2], '0355': [3]}
Let's say I have the following structure:
Map<String,Map<String,Integer>> nestedMap = [
"x": [
"a": 2,
"b": 3,
"c": 4
],
"y": [
"a": 20,
"b": 30,
"c": 40
],
"z": [
"a": 200,
"b": 300,
"c": 400
]
]
I want to flatten this structure and get:
[
"x-a" : 2,
"x-b" : 3,
"x-c" : 4,
"y-a" : 20,
"y-b" : 30,
"y-c" : 40,
"z-a" : 200,
"z-b" : 300,
"z-c" : 400
]
How can I do this with collect/collectEnteries etc?
Alternatively, as you can see the values are quite similar. so the way I have constructed this nested map structure, was by running a for loop iterating through [x,y,z] and setting the entries of my map to be equal to a function that iterates through [a,b,c] and returns a map. Is there a way of skipping this nested structure and functionally append the elements in a big map?
You can create a recursive function that will loop over the map and flatten it.
Map<String,Map<String,Integer>> nestedMap = [
"x": ["a": 2, "b": 3, "z": 4],
"y": ["a": 20, "b": 30, "c": 40],
"z": ["a": 200, "b": 300, "c": 400]
]
String concatenate(String k, String v) {
"${k}-${v}"
}
def flattenMap(Map map) {
map.collectEntries { k, v ->
v instanceof Map ?
flattenMap(v).collectEntries { k1, v1 ->
key = concatenate(k,k1)
[ (key): v1 ]
} :
[ (k): v ]
}
}
print flattenMap(nestedMap)
Regarding the alternate question (and echoing a comment by #damahapatro), consider the following.
Given two lists:
def list1 = ['x', 'y', 'z']
def list2 = ['a', 'b', 'c']
and a function f that gives the appropriate values, then we can use inject (which will accumulate a map while iterating over the combos (e.g. [x,a]):
def m = [list1, list2].combinations().inject([:]) { map, combo ->
assert combo.size() == 2
def a = combo[0]
def b = combo[1]
map["${a}-${b}" as String] = f(a,b)
map
}
assert m == [
"x-a" : 2, "x-b" : 3, "x-c" : 4,
"y-a" : 20, "y-b" : 30, "y-c" : 40,
"z-a" : 200, "z-b" : 300, "z-c" : 400
]
Another way of building the map from scratch would be:
def result = ['a'..'c', 'x'..'z']
.combinations()
.collect { a, b -> b + '-' + a }
.indexed()
.collectEntries { idx, name ->
[name, ((idx % 3) + 2) * (int)Math.pow(10, idx.intdiv(3))]
}
I modified the define syntax of #Paul Walkington and tested it:
String concatenate(String k, String v) {
"${k}-${v}"
}
def flattenMap(Map map) {
map.collectEntries { k, v ->
v instanceof Map ?
flattenMap(v).collectEntries { k1, v1 ->
def key = concatenate(k,k1)
[ (key): v1 ]
} :
[ (k): v ]
}
}
You can do your task with a single istruction:
Map<String,Map<String,Integer>> nestedMap = [
"x": ["a": 2, "b": 3, "c": 4],
"y": ["a": 20, "b": 30, "c": 40],
"z": ["a": 200, "b": 300, "c": 400] ]
result = nestedMap.inject([:]){ a, k, v ->
a << v.collectEntries{ [k + '-' + it.key, it.value] }; a }
println result
(plus setting the source data, plus print the result).