Elasticsearch aggregate unique attribute per groupId - search

Question

I have a separate doc in elasticsearch for each product.
Each product has a unique productId and a non unique groupId along with other attributes eg: categories.
I want to be able to aggregate different attributes with their count per unique groupId
example:
doc 1:
{
"productId": 123
"groupId" xyz,
"categories": [{"value": "shoes"}, {"value": "t-shirt"}]
}
doc 2:
{
"productId": 345
"groupId" xyz,
"categories": [{"value": "shoes"}, {"value": "t-shirt"}]
}
doc 3:
{
"productId": 456
"groupId" abc,
"categories": [{"value": "t-shirt"}]
}
doc 4:
{
"productId": 567
"groupId" abc,
"categories": [{"value": "shoes"}, {"value": "makeup"}]
}
expected results, something like:
shoes: 2
t-shirt: 2
makeup: 1
so I want to count each item once if it exists with the same groupId
my query:
{
"from":0,
"size":0,
"query":{
"filtered":{
"filter":{
}
}
},
"aggs": {
"group": {
"terms": {"field": "group"},
"aggs": {
"brand": {
"terms": {"field": "productMeta.brand.value"}
}
}
}
}
}
response:
{
"took": 6,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 25,
"max_score": 0,
"hits": []
},
"aggregations": {
"group": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 3,
"buckets": [
{
"key": "wlmr34210507",
"doc_count": 8,
"brand": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "generic",
"doc_count": 8
}
]
}
},
{
"key": "wlmr19524441",
"doc_count": 4,
"brand": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "maybelline",
"doc_count": 4
}
]
}
},
{
"key": "wlmr34121549",
"doc_count": 2,
"brand": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "maybelline",
"doc_count": 2
}
]
}
},
{
"key": "wlmr34317301",
"doc_count": 2,
"brand": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "dream on me",
"doc_count": 2
}
]
}
},
{
"key": "bbfs40549552",
"doc_count": 1,
"brand": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "samsung",
"doc_count": 1
}
]
}
},
{
"key": "bobb7937347",
"doc_count": 1,
"brand": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "chicco",
"doc_count": 1
}
]
}
},
{
"key": "wlmr24241413",
"doc_count": 1,
"brand": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "maybelline",
"doc_count": 1
}
]
}
},
{
"key": "wlmr27504560",
"doc_count": 1,
"brand": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "mr. beer",
"doc_count": 1
}
]
}
},
{
"key": "wlmr33986448",
"doc_count": 1,
"brand": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "mr. beer",
"doc_count": 1
}
]
}
},
{
"key": "wlmr40806575",
"doc_count": 1,
"brand": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "healthtex",
"doc_count": 1
}
]
}
}
]
}
}
}

Answer 1

so basically I was able to solve this problem by using cardinality as follows:
{
"from":0,
"size":0,
"query":{
"filtered":{
"filter":{
}
}
},
"sort":{
"ts":{
"order":"desc",
"mode":"max",
"ignore_unmapped":true
}
},
"aggs":{
"categories":{
"terms":{
"field":"productMeta.brand.value",
"size":0
},
"aggs": {
"category" : {
"cardinality" : {
"field" : "group"
}
}
}
}
}
}
the results are unique count per productId per category:
{
"took": 4,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 71,
"max_score": 0,
"hits": []
},
"aggregations": {
"categories": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "chocolate",
"doc_count": 23,
"category": {
"value": 23
}
},
{
"key": "notebook",
"doc_count": 9,
"category": {
"value": 1
}
},
{
"key": "olive_oil",
"doc_count": 7,
"category": {
"value": 7
}
},
{
"key": "physical_training",
"doc_count": 5,
"category": {
"value": 5
}
},
{
"key": "ski",
"doc_count": 5,
"category": {
"value": 2
}
},
{
"key": "gym_membership",
"doc_count": 4,
"category": {
"value": 4
}
},
{
"key": "ski_boots",
"doc_count": 4,
"category": {
"value": 1
}
},
{
"key": "vinegar",
"doc_count": 4,
"category": {
"value": 4
}
},
{
"key": "bracelet",
"doc_count": 3,
"category": {
"value": 3
}
},
{
"key": "handbags",
"doc_count": 2,
"category": {
"value": 2
}
},
{
"key": "cider",
"doc_count": 1,
"category": {
"value": 1
}
},
{
"key": "ice_cider",
"doc_count": 1,
"category": {
"value": 1
}
},
{
"key": "jewelry_1",
"doc_count": 1,
"category": {
"value": 1
}
},
{
"key": "laces",
"doc_count": 1,
"category": {
"value": 1
}
},
{
"key": "stationery",
"doc_count": 1,
"category": {
"value": 1
}
}
]
}
}
}

Answer 2

Using nested elasticsearch Terms Aggregations the same way you are currently doing with brands, you can produce output structured like this:
"aggregations": {
"groups": {
"buckets": [
{
"key": "xyz",
"categories_per_group": {
"buckets": [
{
"key": "shoes",
"doc_count": 2
},
{
"key": "t-shirt",
"doc_count": 2
}
]
}
},
{
"key": "abc",
"categories_per_group": {
"buckets": [
{
"key": "shoes",
"doc_count": 1
},
{
"key": "t-shirt",
"doc_count": 1
},
{
"key": "makeup",
"doc_count": 1
}
]
}
}
]
}
}
It might be possible to write a pipeline bucket script (ES 2.x) to collect the count of distinct categories across group buckets as you're suggesting.
But it's probably simpler and quicker just to implement this reduce logic yourself using the aggregation buckets output above.