Elastic(search): Get docs with max and min timestamp values - search

Question

I got a problem with a search I just can't figure out how to do it. My docs are of the following form:
{
"timestamp":"2015-03-17T15:05:04.563Z",
"session_id":"1",
"user_id":"jan"
}
Let's say the first timestamp of a session id is the "Login" and the last timestamp is the "Logout". I want to have all "login" and "logout" docs for all sessions (if possible sorted by user_id). I managed to get the right timestamps with aggregations:
{
"aggs" : {
"group_by_uid" : {
"terms" : {
"field" : "user_id"
},
"aggs" : {
"group_by_sid" : {
"terms" : {
"field" : "session_id"
},
"aggs" : {
"max_date" : {
"max": { "field" : "timestamp" }
},
"min_date" : {
"min": { "field" : "timestamp" }
}
}
}
}
}
}
}
But how do I get the corresponding docs? I also don't mind if i have to do 2 searches (one for the logins and one for the logouts). I tried tome top hits aggregations and sorting stuff but I always get parse errors :/
I hope someone can give me a hint :)
Best regards,
Jan

Answer 1

Here's a solution in a single search based on the approach proposed by Sloan Ahrens. The advantage is that the start and end session entries are in the same bucket.
{
"aggs": {
"group_by_uid": {
"terms": {
"field": "user_id"
},
"aggs": {
"group_by_sid": {
"terms": {
"field": "session_id"
},
"aggs": {
"session_start": {
"top_hits": {
"size": 1,
"sort": [ { "timestamp": { "order": "asc" } } ]
}
},
"session_end": {
"top_hits": {
"size": 1,
"sort": [ { "timestamp": { "order": "desc" } } ]
}
}
}
}
}
}
}
}
Cheers,
Jan

Answer 2

You're already close. How about this. Use two searches, each aggregating the way you did, but then also get the first top_hit sorting on "timestamp".
I just set up a basic index and added some data that looks like what you posted:
PUT /test_index
{
"settings": {
"number_of_shards": 1
}
}
POST /test_index/_bulk
{"index":{"_index":"test_index","_type":"doc","_id":1}}
{"timestamp":"2015-03-17T15:05:04.563Z","session_id":"1","user_id":"jan"}
{"index":{"_index":"test_index","_type":"doc","_id":2}}
{"timestamp":"2015-03-17T15:10:04.563Z","session_id":"1","user_id":"jan"}
{"index":{"_index":"test_index","_type":"doc","_id":3}}
{"timestamp":"2015-03-17T15:15:04.563Z","session_id":"1","user_id":"jan"}
{"index":{"_index":"test_index","_type":"doc","_id":4}}
{"timestamp":"2015-03-17T18:05:04.563Z","session_id":"1","user_id":"bob"}
{"index":{"_index":"test_index","_type":"doc","_id":5}}
{"timestamp":"2015-03-17T18:10:04.563Z","session_id":"1","user_id":"bob"}
{"index":{"_index":"test_index","_type":"doc","_id":6}}
{"timestamp":"2015-03-17T18:15:04.563Z","session_id":"1","user_id":"bob"}
Then I can get each session's start time with:
POST /test_index/_search?search_type=count
{
"aggs": {
"group_by_uid": {
"terms": {
"field": "user_id"
},
"aggs": {
"group_by_sid": {
"terms": {
"field": "session_id"
},
"aggs": {
"session_start": {
"top_hits": {
"size": 1,
"sort": [ { "timestamp": { "order": "asc" } } ]
}
}
}
}
}
}
}
}
...
{
"took": 5,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 6,
"max_score": 0,
"hits": []
},
"aggregations": {
"group_by_uid": {
"buckets": [
{
"key": "bob",
"doc_count": 3,
"group_by_sid": {
"buckets": [
{
"key": "1",
"doc_count": 3,
"session_start": {
"hits": {
"total": 3,
"max_score": null,
"hits": [
{
"_index": "test_index",
"_type": "doc",
"_id": "4",
"_score": null,
"_source": {
"timestamp": "2015-03-17T18:05:04.563Z",
"session_id": "1",
"user_id": "bob"
},
"sort": [
1426615504563
]
}
]
}
}
}
]
}
},
{
"key": "jan",
"doc_count": 3,
"group_by_sid": {
"buckets": [
{
"key": "1",
"doc_count": 3,
"session_start": {
"hits": {
"total": 3,
"max_score": null,
"hits": [
{
"_index": "test_index",
"_type": "doc",
"_id": "1",
"_score": null,
"_source": {
"timestamp": "2015-03-17T15:05:04.563Z",
"session_id": "1",
"user_id": "jan"
},
"sort": [
1426604704563
]
}
]
}
}
}
]
}
}
]
}
}
}
and end-time with:
POST /test_index/_search?search_type=count
{
"aggs": {
"group_by_uid": {
"terms": {
"field": "user_id"
},
"aggs": {
"group_by_sid": {
"terms": {
"field": "session_id"
},
"aggs": {
"session_end": {
"top_hits": {
"size": 1,
"sort": [ { "timestamp": { "order": "desc" } } ]
}
}
}
}
}
}
}
}
...
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 6,
"max_score": 0,
"hits": []
},
"aggregations": {
"group_by_uid": {
"buckets": [
{
"key": "bob",
"doc_count": 3,
"group_by_sid": {
"buckets": [
{
"key": "1",
"doc_count": 3,
"session_end": {
"hits": {
"total": 3,
"max_score": null,
"hits": [
{
"_index": "test_index",
"_type": "doc",
"_id": "6",
"_score": null,
"_source": {
"timestamp": "2015-03-17T18:15:04.563Z",
"session_id": "1",
"user_id": "bob"
},
"sort": [
1426616104563
]
}
]
}
}
}
]
}
},
{
"key": "jan",
"doc_count": 3,
"group_by_sid": {
"buckets": [
{
"key": "1",
"doc_count": 3,
"session_end": {
"hits": {
"total": 3,
"max_score": null,
"hits": [
{
"_index": "test_index",
"_type": "doc",
"_id": "3",
"_score": null,
"_source": {
"timestamp": "2015-03-17T15:15:04.563Z",
"session_id": "1",
"user_id": "jan"
},
"sort": [
1426605304563
]
}
]
}
}
}
]
}
}
]
}
}
}
Here's the code I used:
http://sense.qbox.io/gist/05edb48b840e6a992646643913db8ef0a3ccccb3