I'm trying to retrieve some data from ElasticSearch.
So far everything is working perfectly and I can query data.
But whenever I try to count a field using aggregations, the aggregation field is not in the result at the end.
So far what I've tried this as my query/function :
var client = new elasticsearch.Client({
host: 'xxxxxxxxxxxxxxxxxxxxxxx',
log:"trace"
});
client.ping({
requestTimeout: 30000,
}, function (error) {
if (error) {
console.error('elasticsearch cluster is down!');
} else {
console.log('All is well');
}
});
client.search({
"index":"worklight__appsession__1485302400000",
"type":"AppSession",
"body":{
"query": {
"filtered": {
"query": {
"query_string": {
"analyze_wildcard": true,
"query": "*"
}
},
"filter": {
"bool": {
"must": [
{
"range": {
"timestamp": {
"gte": 1553507131976,
"lte": 1553593531976
}
}
}
],
"must_not": []
}
}
}
},
"aggs": {
"1": {
"cardinality": {
"field": "deviceID"
}
}
}
}
}).then(function (resp) {
var hits = resp.hits.hits;
console.log(hits)
}, function (err) {
console.trace(err.message);
});
and the result is :
Elasticsearch DEBUG: 2019-03-26T09:46:21Z
starting request {
"method": "HEAD",
"requestTimeout": 30000,
"castExists": true,
"path": "/",
"query": {}
}
Elasticsearch DEBUG: 2019-03-26T09:46:21Z
starting request {
"method": "POST",
"path": "/worklight__appsession__1485302400000/AppSession/_search",
"body": {
"query": {
"filtered": {
"query": {
"query_string": {
"analyze_wildcard": true,
"query": "*"
}
},
"filter": {
"bool": {
"must": [
{
"range": {
"timestamp": {
"gte": 1553507131976,
"lte": 1553593531976
}
}
}
],
"must_not": []
}
}
}
},
"aggs": {
"1": {
"cardinality": {
"field": "deviceID"
}
}
}
},
"query": {}
}
Elasticsearch TRACE: 2019-03-26T09:46:22Z
-> HEAD http://xx/
<- 200
Elasticsearch DEBUG: 2019-03-26T09:46:22Z
Request complete
All is well
Elasticsearch TRACE: 2019-03-26T09:46:22Z
-> POST http://xx/worklight__appsession__1485302400000/AppSession/_search
{
"query": {
"filtered": {
"query": {
"query_string": {
"analyze_wildcard": true,
"query": "*"
}
},
"filter": {
"bool": {
"must": [
{
"range": {
"timestamp": {
"gte": 1553507131976,
"lte": 1553593531976
}
}
}
],
"must_not": []
}
}
}
},
"aggs": {
"1": {
"cardinality": {
"field": "deviceID"
}
}
}
}
<- 200
{
"took": 4,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 325,
"max_score": 1,
"hits": [
...
confidential data here, not relevant to the topic.
...
}
]
},
"aggregations": {
"1": {
"value": 133
}
}
}
But if erase the log trace option, aggregations don't show up in the result :
[ { _index: 'worklight__appsession__1485302400000',
_type: 'AppSession',
... Some Data,
{ _index: 'worklight__appsession__1485302400000',
_type: 'AppSession',
... Some Data,
{ _index: 'worklight__appsession__1485302400000',
_type: 'AppSession',
... Some Data,
]
Am I doing something wrong, or do I just lack knowledge ?
Thanks for your time.
You are doing console.log(resp.hits.hits). Try this instead:
console.log(resp.aggregations)
Related
I want to find duplicate values and if there are duplicate values then I sort based on the last update, so what I take is the newest one, how do I do aggregations? I've tried this aggregation.
I've tried adding sort to sources but it still doesn't work, I've tried several ways but it still fails sometimes it comes out 1 but only old data, sometimes the order is correct from the newest but appears 2 data
{
"size": 0,
"query": {
"bool": {
"must": [
{
"match": {
"BILLING_TYPE_CD": "Service Bundle"
}
},
{
"match": {
"ID": "xxxx"
}
},
{
"exists": {
"field": "LI_MILESTONE"
}
},
{
"exists": {
"field": "LI_SID"
}
},
{
"query_string": {
"default_field": "LI_SID",
"query": "*xxxx*"
}
}
],
"must_not": {
"bool": {
"must": [
{
"query_string": {
"default_field": "LI_PRODUCT_NAME",
"query": "*Network*"
}
},
{
"terms": {
"LI_MILESTONE.keyword": [
"Abandoned",
"Cancelled"
]
}
},
{
"terms": {
"ORDER_STATUS.keyword": [
"Abandoned",
"Cancelled",
"Drop In Progress"
]
}
},
{
"term": {
"STATUS.keyword": ""
}
}
]
}
}
}
},
"sort": [
{
"TGL_CREATED": {
"order": "desc"
}
}
],
"aggs": {
"list_products": {
"composite": {
"size": 50000,
"sources": [
{
"LI_SID": {
"terms": {
"field": "LI_SID.keyword",
"order": "desc"
}
}
}
]
},
"aggs": {
"totalService": {
"terms": {
"field": "LI_SID.keyword",
"size": 50000,
"order": {
"_term": "asc"
}
}
},
"bucket_sort": {
"bucket_sort": {
"from": 0,
"size": 10
}
},
"includes_source": {
"top_hits": {
"size": 1,
"_source": {
"includes": [
"LAST_UPDATE",
"xxxxx",
"xxxxx",
"xxxxx",
"xxx"
]
}
}
}
}
},
"term_product": {
"terms": {
"field": "LI_SID.keyword",
"size": 50000
}
}
}
}
Like this ?
{
"aggs": {
"LI_SID": {
"terms": {
"field": "LI_SID.keyword",
"size": 10
},
"aggs": {
"hit": {
"top_hits": {
"size": 1,
"sort": [
{
"LAST_UPDATE": "desc"
}
]
}
}
}
}
},
"size": 0
}
You need to use aggregations response not hits
I have following document in ES :
[
{
"event_id": 123,
"event_name": "test event",
"event_date": "2018-12-21",
"ticket_group": [
{
"available": 8,
"price": 8,
"id": "159831",
"parking_passes_available": 0,
"field_values": [
{
"field_id": 589,
"field_value": "KUMAR"
},
{
"field_id": 717,
"field_value": "AMIT"
},
{
"field_id": 1360,
"field_value": "SAM"
},
{
"field_id": 2239,
"field_value": ""
},
{
"field_id": 2240,
"field_value": ""
},
{
"field_id": 2241,
"field_value": ""
},
{
"field_id": 2242,
"field_value": ""
}
]
}
]
}
]
and i want to search with multiple field_id and field_value with AND operator. But it works if there is single condition but not for multiple cases. Here is what i have tried so far :
{
"query": {
"bool": {
"must": [
{
"nested": {
"path": "ticket_group",
"score_mode": "max",
"inner_hits": {
"from": 0,
"size": 10000
},
"query": {
"bool": {
"must": [
{
"nested": {
"path": "ticket_group.field_values",
"score_mode": "max",
"inner_hits": {
"from": 0,
"size": 10000
},
"query": {
"bool": {
"must": [
{
"bool": {
"must": [
{
"match": {
"ticket_group.field_values.field_id": 589
}
},
{
"match": {
"ticket_group.field_values.field_value": "KUMAR"
}
}
]
}
},
{
"bool": {
"must": [
{
"match": {
"ticket_group.field_values.field_id": 717
}
},
{
"match": {
"ticket_group.field_values.field_value": "AMIT"
}
}
]
}
}
]
}
}
}
}
]
}
}
}
}
]
}
},
"size": 10,
"from": 0,
"sort": {
"event_date": {
"order": "asc"
}
}
}
i want to search ticket group if field_id=717 with value "amit" and field_id=589 with value "kumar" exists in field_values object inside ticket_group. Using above query i am getting no records while objects with both values exist in field_values.
Can anyone help to build a such query ?
Thank You
Below is what you are looking for. You simply need to push the second level nested into two must clauses.
POST <your_index_name>/_search
{
"query":{
"bool":{
"must":[
{
"nested":{
"path":"ticket_group",
"score_mode":"max",
"inner_hits":{
},
"query":{
"bool":{
"must":[
{
"nested":{
"path":"ticket_group.field_values",
"score_mode":"max",
"inner_hits":{
"name":"inner_clause_1"
},
"query":{
"bool":{
"must":[
{
"match":{
"ticket_group.field_values.field_id":589
}
},
{
"match":{
"ticket_group.field_values.field_value":"KUMAR"
}
}
]
}
}
}
},
{
"nested":{
"path":"ticket_group.field_values",
"score_mode":"max",
"inner_hits":{
"name":"inner_clause_2"
},
"query":{
"bool":{
"must":[
{
"match":{
"ticket_group.field_values.field_id":717
}
},
{
"match":{
"ticket_group.field_values.field_value":"AMIT"
}
}
]
}
}
}
}
]
}
}
}
}
]
}
}
}
Notice that I've named the inner_hits in the second level nested queries.
If you don't do that(try by removing the name key in the inner_hits), then you would only see the inner_hit for the last clause which ends up overwriting the inner_hits result of the first nested clause.
Hope this helps!
I have a request into Elastic
{
"query":{
"bool":{
"must":[
{
"query_string":{
"query":"something1 OR something2 OR something3",
"default_operator":"OR"
}
}
],
"filter":{
"range":{
"time":{
"gte":date
}
}
}
}
}
}
I wanna calculate count for each token in all documents using elastic search in one request, for example:
something1: 26 documents
something2: 12 documents
something3: 1 documents
Assuming that the tokens are not akin to enumerations (i.e. constrained set of specific values, like state names, which would make a terms aggregation your best bet with the right mapping), I think the closest thing to what you want would be to use filters aggregation:
POST your-index/_search
{
"query":{
"bool":{
"must":[
{
"query_string":{
"query":"something1 OR something2 OR something3",
"default_operator":"OR"
}
}
],
"filter":{
"range":{
"time":{
"gte":date
}
}
}
}
},
"aggs": {
"token_doc_counts": {
"filters" : {
"filters" : {
"something1" : {
"bool": {
"must": { "query_string" : { "query" : "something1" } },
"filter": { "range": { "time": { "gte": date } } }
}
},
"something2" : {
"bool": {
"must": { "query_string" : { "query" : "something2" } },
"filter": { "range": { "time": { "gte": date } } }
}
},
"something3" : {
"bool": {
"must": { "query_string" : { "query" : "something3" } },
"filter": { "range": { "time": { "gte": date } } }
}
}
}
}
}
}
}
The response would look something like:
{
"took": 9,
"timed_out": false,
"_shards": ...,
"hits": ...,
"aggregations": {
"token_doc_counts": {
"buckets": {
"something1": {
"doc_count": 1
},
"something2": {
"doc_count": 2
},
"something3": {
"doc_count": 3
}
}
}
}
}
You can split your query into filters aggregation of three filters. For reference look here: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-filters-aggregation.html
What you would need to do, is to create a Copy_To field and have the mapping as shown below.
Depending on the fields that your query_string queries, you need to include some or all of the fields with copy_to field.
By default query_string searches all the fields, so you may need to specify copy_to for all the fields as shown in below mapping, where for sake of simplicity, I've created only three fields, title, field_2 and a third field content which would act as copied to field.
Mapping
PUT <your_index_name>
{
"mappings": {
"mydocs": {
"properties": {
"title": {
"type": "text",
"copy_to": "content"
},
"field_2": {
"type": "text",
"copy_to": "content"
},
"content": {
"type": "text",
"fielddata": true
}
}
}
}
}
Sample Documents
POST <your_index_name>/mydocs/1
{
"title": "something1",
"field_2": "something2"
}
POST <your_index_name>/mydocs/2
{
"title": "something2",
"field_2": "something3"
}
Query:
You'd get the required document counts for the each and every token using the below aggregation query and I've made use of Terms Aggregation:
POST <your_index_name>/_search
{
"size": 0,
"query": {
"query_string": {
"query": "something1 OR something2 OR something3"
}
},
"aggs": {
"myaggs": {
"terms": {
"field": "content",
"include" : ["something1","something2","something3"]
}
}
}
}
Query Response:
{
"took": 7,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 2,
"max_score": 0,
"hits": []
},
"aggregations": {
"myaggs": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "something2",
"doc_count": 2
},
{
"key": "something1",
"doc_count": 1
},
{
"key": "something3",
"doc_count": 1
}
]
}
}
}
Let me know if it helps!
I have 20 documents and i'm performing aggregation based on reportid. I need top 10 aggregation based on time in descending. But the response is very random. What am i missing? I'm using elasticsearch 6.2.2 and node.js 4.5. Below here is the body search query for elasticsearch request.
{
"size": 0,
"sort": [
{
"triggerDate":
{
"order": "desc"
}
}],
"query":
{
"bool":
{
"must": [
{
"query_string":
{
"query": "*",
"analyze_wildcard": true
}
},
{
"range":
{
"triggerDate":
{
"gte": fromTime,
"lte": toTime
}
}
}
],
"must_not": [
{
"query_string":
{
"query": "reportId.keyword:\"\"",
"analyze_wildcard": true
}
}]
}
},
"_source":
{
"excludes": []
},
"aggs":
{
"reportid":
{
"terms":
{
"field": "reportId.keyword",
"size": 10
}
}
}
I think what you need to do is aggregate on reportId.keyword and sort aggregation by date.
So here is the solution
{
"size": 0,
"query": {
"bool": {
"must": [
{
"query_string": {
"query": "*",
"analyze_wildcard": true
}
},
{
"range": {
"triggerDate": {
"gte": fromTime,
"lte": toTime
}
}
}
],
"must_not": [
{
"query_string": {
"query": "reportId.keyword:\"\"",
"analyze_wildcard": true
}
}
]
}
},
"_source": {
"excludes": []
},
"aggs": {
"reportid": {
"terms": {
"field": "reportId.keyword",
"size": 10,
"order": {
"2-orderAgg": "desc"
}
},
"aggs": {
"2-orderAgg": {
"max": {
"field": "triggerDate"
}
}
}
}
}
}
You need to sort the aggregation results by a custom aggregation and not the query results.
I'm getting started with ElasticSearch and I'm getting into troubles (I'm not understanding as It has to be) how to search.
First, I have this two documents:
{
"took": 133
"timed_out": false
"_shards": {
"total": 5
"successful": 5
"failed": 0
}
"hits": {
"total": 2
"max_score": 1
"hits": [2]
0: {
"_index": "app"
"_type": "player"
"_id": "AVcLCOgAi_gt2Fih02MK"
"_score": 1
"_source": {
"nickName": "sarasa"
"birthDate": "1994-11-05T13:15:30.000Z"
"state": "sarasa"
"adminState": "sarasa"
"id": ""
"account": {
"type": "yojuego"
"id": "asasdfa"
"password": "asd fasd"
}
}
}
1: {
"_index": "app"
"_type": "player"
"_id": "AVcQ7JNVi_gt2Fih02MN"
"_score": 1
"_source": {
"nickName": "facundo"
"birthDate": "1994-11-05T13:15:30.000Z"
"state": "verdura"
"adminState": "sudo"
"id": ""
"account": {
"type": "yojuego"
"id": "facundo#facundo"
"password": "pepe"
}
}
}
}
}
}
I want to get where account.id = "facundo#facundo" and account.type = "yojuego".
I'm doing this:
client.search({
index: 'app',
type: 'player',
query: {
bool: {
must: [
{ term: { "account.id": 'facundo#facundo' } },
{ term: { "account.type": 'yojuego' } }
],
}
}
}, (error, response, status) => {
if (error) {
res.json(400, err);
}
else {
res.json(200, response.hits.hits);
}
});
This search is retrieving all documents I have into the index.
Any help?
Thanks!
PD: Here is how I created index and mapping:
client.indices.create({ index: 'yojuego' }, (err, resp, respcode) => {
if (!err) {
client.indices.putMapping({
index: 'app',
type: "player",
body: {
properties: {
nickName: { type: "string" },
birthDate: { type: "string" },
state: { type: "string" },
adminState: { type: "string" },
account: {
type: "nested",
properties: {
id: { type: "string" },
type: { type: "string" },
password: { type: "string" }
}
}
}
}
}, (err, resp, respcode) => {
res.json(200, resp);
});
}
});
make sure that account is a nested field and then apply this query,
{
"query": {
"bool": {
"must": [
{
"nested": {
"path": "account",
"query": {
"bool": {
"must": [
{
"match": {
"account.id": "facundo#facundo"
}
},
{
"match": {
"account.type": "yojuego"
}
}
]
}
}
}
}
]
}
}
}