No results match your search criteria in kibana with timestamp - node.js

I created an index in elasticsearch 6.5.1 successfully loaded the data to that index. there is one field "submitted_date" which is the timestamp. below is the mapping like of this field.
"submitted_date": { "type": "date", "format":"yyyy-MM-dd HH:mm:ss.SSS" },
then I created the index pattern. I used the Time Filter field name as "submitted_date". after that, I tried to check the data in Discover tab, but data are not showing. there is a message saying that No results match your search criteria.
NOTE that I have changed the time in time range picker in every possible way which is on top of the right corner in kibana dashboard.
data appear in Dev Tools tab with elastic queries.
ps : I inserted the data using nodejs with elasticsearch official library, did not used logstash.
I followed this article, but it did not help me.
UPDATE : sample document
{
"took" : 0,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : 10480,
"max_score" : 1.0,
"hits" : [
{
"_index" : "test",
"_type" : "tests",
"_id" : "1214334",
"_score" : 1.0,
"_source" : {
"priority" : "4",
"submitted_date" : "2018-01-04T18:32:21.000Z",
"submitted_month" : 0,
"submitted_month_name" : "January",
"submitted_day" : 4,
"submitted_weekday" : "Tuesday",
"submitted_hour" : 18,
"submitted_year_month" : "2018-0",
"submitted_year_month_name" : "2018-January",
"date_key" : "20180104",
"year_month_key" : "201801",
"status" : "Closed"
}
}
]
}
}
Inspect request
{
"version": true,
"size": 500,
"sort": [
{
"_score": {
"order": "desc"
}
}
],
"_source": {
"excludes": []
},
"aggs": {
"2": {
"date_histogram": {
"field": "submitted_date",
"interval": "1d",
"time_zone": "Asia/Kolkata",
"min_doc_count": 1
}
}
},
"stored_fields": [
"*"
],
"script_fields": {},
"docvalue_fields": [
{
"field": "close_date",
"format": "date_time"
},
{
"field": "last_modified_date",
"format": "date_time"
},
{
"field": "last_resolved_date",
"format": "date_time"
},
{
"field": "submitted_date",
"format": "date_time"
},
{
"field": "time_to_resolve",
"format": "date_time"
}
],
"query": {
"bool": {
"must": [
{
"match_all": {}
},
{
"range": {
"submitted_date": {
"gte": 1514745000000,
"lte": 1543937620414,
"format": "epoch_millis"
}
}
}
],
"filter": [],
"should": [],
"must_not": []
}
},
"highlight": {
"pre_tags": [
"#kibana-highlighted-field#"
],
"post_tags": [
"#/kibana-highlighted-field#"
],
"fields": {
"*": {}
},
"fragment_size": 2147483647
}
}
Index pattern
function _putMapping() {
return client.indices.create({
index: process.env.ELASTICSEARCH_INDEX,
body: {
settings:{
index:{
"number_of_shards": 1,
"number_of_replicas": 5
},
"index.mapping.ignore_malformed" : true
},
mappings:{
tests:{
properties:{
"last_modified_date": { "type": "date" },
"last_resolved_date": { "type": "date" },
"time_to_resolve": { "type": "date" },
"submitted_date": { "type": "date", "format":"yyyy-MM-dd HH:mm:ss.SSS" },
"date_key": { "type": "integer" },
"priority": { "type": "long" },
"submitted_hour": { "type": "long" },
"submitted_month": { "type": "long" },
"submitted_year": { "type": "long" },
"submitted_year": { "type": "keyword" },
"submitted_year_month": { "type": "keyword" },
"submitted_year_month_name": { "type": "keyword" },
}
}
}
}
});
}

Your mYour submitted_date is coming like 2018-01-04T18:32:21.000Z but your mapping is set as yyyy-MM-dd HH:mm:ss.SSS.
You need to change it to "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'".

Related

Elasticsearch query including documents it should exclude and vice versa

IMPORTANT EDIT: The logic described below appears to be all correct. The root of my issue was actually being caused by the utility we used to push new data to the ES database, not with the query itself. I have accepted the answer that says that the query works as intended.
I have an Elasticsearch server whose mapping looks like this (as output by curl 'elastic:9200/resourcelibrary/_mapping):
{
"resourcelibrary": {
"mappings": {
"resource": {
"properties": {
"created_at": {
"type": "date"
},
"created_by": {
"type": "text"
},
"custom_key": {
"type": "keyword"
},
"defaultAction": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"default_action": {
"type": "keyword"
},
"description": {
"type": "text"
},
"id": {
"type": "text"
},
"indexed": {
"type": "keyword"
},
"is_searchable": {
"type": "keyword"
},
"key": {
"type": "text"
},
"licenses": {
"type": "keyword"
},
"raw": {
"type": "text"
},
"require_priv": {
"type": "keyword"
},
"source": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
},
"fielddata": true
},
"stat": {
"type": "text"
},
"style_def": {
"type": "keyword"
},
"tags": {
"type": "text"
},
"thumbnail": {
"type": "text"
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "keyword"
},
"uid": {
"type": "text"
},
"updated_at": {
"type": "date"
},
"updated_by": {
"type": "text"
}
}
}
}
}
}
With the resourcelibrary collection completely empty, I add the following documents to it:
[
{
'type' : 'video',
'uid' : '2c444278-e0d3-497b-9b5b-b70756b0fdc0',
'key' : 'test-test',
'custom_key' : 'test-test',
'description' : 'Random text just to fill up the description. Also, math',
'privileged' : [],
'require_priv' : true,
'title' : 'Title!!!',
'defaultAction': '9dfcdb39-6644-4023-82c3-8227ba184c02',
'source' : 'frontline'
},
{
'type' : 'course',
'uid' : '8afb5c95-c7b5-498a-abec-ae829d164964',
'key' : 'test-scorm',
'custom_key' : 'test-scorm',
'description' : 'SCORM!!!',
'privileged' : [],
'require_priv' : true,
'title' : 'SCORM!!!',
'defaultAction': '1302dead-9941-4b90-b35c-30eff4993365',
'source' : 'scormcloud'
},
{
'type' : 'mc',
'uid' : '8e66c2fa-6090-49da-91dd-d939124fef90',
'key' : 'test-mc',
'custom_key' : 'test-mc',
'description' : 'MC!!!',
'require_priv' : false,
'title' : 'MC!!!',
'defaultAction': '7957b8f5-c934-4296-b7bb-70f2cc4b2ad0',
'source' : 'edivate'
},
{
'type' : 'group',
'uid' : '80a908c3-dd6c-4902-9f05-647a8af689ac',
'key' : 'test-group',
'custom_key' : 'test-group',
'description' : 'GROUP!!!',
'require_priv' : false,
'title' : 'GROUP!!!',
'defaultAction': '25b700a5-7563-4d6e-9eab-18465d08a683',
'source' : 'two words'
},
{
'type' : 'video',
'uid' : '3d555389-e0d3-497b-9b5b-c81867c10ed1',
'key' : 'test-video',
'custom_key' : 'test-video',
'description' : 'Random text just to fill up the description. Also, science',
'require_priv' : false,
'title' : 'NO-PRIVS-REQUIRED RESOURCE!!!',
'defaultAction': '9dfcdb39-6644-4023-82c3-8227ba184c02',
'source' : 'ets'
},
{
'type' : 'video',
'uid' : 'fbc0f853-9020-4ed7-8d4d-e18ebe75d815',
'key' : 'test-test-test',
'custom_key' : 'test-test-test',
'description' : 'integration testing description',
'require_priv' : false,
'title' : 'Search All Resources Integration Title',
'defaultAction': '9dfcdb39-6644-4023-82c3-8227ba184c02'
},
{
'type' : 'file',
'uid' : 'cf84e252-1082-4a94-9fe5-45fa73364e2f',
'key' : 'test-test-test two',
'custom_key' : 'test-test-test two',
'description' : 'integration testing description two',
'require_priv' : false,
'title' : 'Search All Resources Integration Title two',
'defaultAction': '0b7abf9e-c88a-4d19-891d-52fe0b220506'
},
{
'id' : 'ba462b70-de73-4173-88bf-66bc9d1385b9',
'type' : 'course',
'uid' : 'scormcloud-course-cf84e252-1082-4a94-9fe5-45fa73364e2f',
'key' : 'test-test-test two',
'custom_key' : 'test-test-test two',
'description' : 'integration testing description two',
'require_priv' : false,
'title' : 'Search All Resources Integration Title two',
'defaultAction': '0b7abf9e-c88a-4d19-891d-52fe0b220506'
},
{
'id' : '7f0cbbc6-a1dd-43ca-9108-b31f90904dce',
'type' : 'course',
'uid' : 'scormcloud-course-7f0cbbc6-a1dd-43ca-9108-b31f90904dce',
'key' : 'LD_7f0cbbc6-a1dd-43ca-9108-b31f90904dce',
'custom_key' : 'LD_7f0cbbc6-a1dd-43ca-9108-b31f90904dce',
'description' : 'This is a LearningDesigner course',
'require_priv' : false, // Still requires LCR Tooling
'title' : 'LearningDesigner Course 1',
'defaultAction': '1302dead-9941-4b90-b35c-30eff4993365'
},
{
'id' : '9e195a62-1a53-42c0-8648-4aa35c309d48',
'type' : 'course',
'uid' : 'scormcloud-course-9e195a62-1a53-42c0-8648-4aa35c309d48',
'key' : 'user-SCORM_9e195a62-1a53-42c0-8648-4aa35c309d48',
'custom_key' : 'user-SCORM_9e195a62-1a53-42c0-8648-4aa35c309d48',
'description' : 'This is a user-uploaded SCORM course',
'require_priv' : false, // Still requires LCR Tooling
'title' : 'User-Uploaded Course 1',
'defaultAction': '1302dead-9941-4b90-b35c-30eff4993365'
},
{
'id' : '2b1b2197-48ae-4669-8bfa-7edd440cb027',
'type' : 'course',
'uid' : 'course-2b1b2197-48ae-4669-8bfa-7edd440cb027',
'source' : 'canvas',
'key' : '9e195a62-1a53-42c0-8648-4aa35c309d48',
'custom_key' : '9e195a62-1a53-42c0-8648-4aa35c309d48',
'description' : 'This is a Canvas course',
'require_priv' : false,
'title' : 'Canvas Course 1',
'defaultAction': '7aef5ef2-e0c1-4188-9e74-3e24057e7e6e'
},
{
'id' : '5e113295-f905-4a3d-97e0-f5d49926c979',
'type' : 'collaborative',
'uid' : 'frontline-collaborative-5e113295-f905-4a3d-97e0-f5d49926c979',
'key' : '5e113295-f905-4a3d-97e0-f5d49926c979',
'custom_key' : '5e113295-f905-4a3d-97e0-f5d49926c979',
'description' : 'This is a Collaborative resource',
'require_priv' : false,
'title' : 'Collab Resource 1',
'defaultAction': 'e153409d-3330-4202-baf2-d602b4cb7d66'
},
]
I am working with BodyBuilder.JS to produce a query that returns everything EXCEPT for documents that meets any of the following criteria:
type is equal to "collaborative"
source is equal to "canvas"
type is equal to "course" AND custom_key starts with "LD_"
type is equal to "course" AND custom_key starts with "user-SCORM_"
In my application, I implemented these exclusive conditions like so (note: body is an object created with the bodybuilder() function from BodyBuilder.JS):
body.notFilter('term', 'type', 'collaborative');
body.notFilter('term', 'source', 'canvas');
body.notFilter('bool', subFilter => {
subFilter.filter('term', 'type', 'course');
subFilter.filter('regexp', 'custom_key', 'LD_.*');
return subFilter;
});
body.notFilter('bool', subFilter => {
subFilter.filter('term', 'type', 'course');
subFilter.filter('regexp', 'custom_key', 'user-SCORM_.*');
return subFilter;
});
... And when body.build() is called, it produces the following Elasticsearch DSL string:
{
"from": "0",
"size": 100,
"query": {
"bool": {
"filter": {
"bool": {
"must": {
"bool": {}
},
"must_not": [
{
"term": {
"type": "collaborative"
}
},
{
"term": {
"source": "canvas"
}
},
{
"bool": {
"must": [
{
"term": {
"type": "course"
}
},
{
"regexp": {
"custom_key": "LD_.*"
}
}
]
}
},
{
"bool": {
"must": [
{
"term": {
"type": "course"
}
},
{
"regexp": {
"custom_key": "user-SCORM_.*"
}
}
]
}
}
]
}
}
}
}
}
When I send this query to my Elasticsearch server, it correctly omits the documents with type of "collaborative" and source of "canvas". However, it is doing the following incorrect things:
search results are missing the second document (uid value "8afb5c95-c7b5-498a-abec-ae829d164964")
search results are improperly including documents whose type is "course" and whose custom_key starts with "LD_"
search results are improperly including documents whose type is "course" and whose custom_key starts with "user-SCORM_"
I'm genuinely not sure what I'm doing wrong here. I have also tried replacing the regexp subclauses with match subclauses (e.g. subFilter.filter('match', 'custom_key', 'LD_*'),
but I get the exact same results.
I've been staring at this thing for a literal week now, trying hundreds of slightly-different things similar to what is described in this post but this is the closest I've gotten to getting the results I need. What am I doing wrong?
This might help to solve & analyze your problem locally.
I have created the index:
PUT /resourcelibrary
{
"mappings": {
"properties": {
"created_at": {
"type": "date"
},
"created_by": {
"type": "text"
},
"custom_key": {
"type": "keyword"
},
"defaultAction": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"default_action": {
"type": "keyword"
},
"description": {
"type": "text"
},
"id": {
"type": "text"
},
"indexed": {
"type": "keyword"
},
"is_searchable": {
"type": "keyword"
},
"key": {
"type": "text"
},
"licenses": {
"type": "keyword"
},
"raw": {
"type": "text"
},
"require_priv": {
"type": "keyword"
},
"source": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
},
"fielddata": true
},
"stat": {
"type": "text"
},
"style_def": {
"type": "keyword"
},
"tags": {
"type": "text"
},
"thumbnail": {
"type": "text"
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "keyword"
},
"uid": {
"type": "text"
},
"updated_at": {
"type": "date"
},
"updated_by": {
"type": "text"
}
}
}
}
Data Ingestion:
POST /resourcelibrary/_doc
{
"custom_key": "5e113295-f905-4a3d-97e0-f5d49926c979",
"defaultAction": "e153409d-3330-4202-baf2-d602b4cb7d66",
"description": "This is a Collaborative resource",
"id": "5e113295-f905-4a3d-97e0-f5d49926c979",
"key": "5e113295-f905-4a3d-97e0-f5d49926c979",
"require_priv": false,
"title": "Collab Resource 1",
"type": "collaborative",
"uid": "frontline-collaborative-5e113295-f905-4a3d-97e0-f5d49926c979"
}
Search Query (working fine)
GET /resourcelibrary/_search
{
"query": {
"bool": {
"filter": {
"bool": {
"must": {
"bool": {}
},
"must_not": [
{
"term": {
"type": "collaborative"
}
},
{
"term": {
"source": "canvas"
}
},
{
"bool": {
"must": [
{
"term": {
"type": "course"
}
},
{
"regexp": {
"custom_key": "LD_.*"
}
}
]
}
},
{
"bool": {
"must": [
{
"term": {
"type": "course"
}
},
{
"regexp": {
"custom_key": "user-SCORM_.*"
}
}
]
}
}
]
}
}
}
}
}
Try to create an index with different name & try to execute these queries. This way you can debug into actual issue.
But the Query is being generated properly.

Elastic Search multi match query can't ignore special characters

I have a name field value as "abc_name" so when I search "abc_" I am getting proper results but when I search "abc_##£&-#&" still I am getting same results. I want my query to ignore this special characters that doesn't matches with my query.
My query has:
Multi_match
type as cross_fields
operator AND
I am using search_analyzer standard for my Fields
And I want this structure as it is otherwise it will affect my other Search behaviour
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "autocomplete",
"search_analyzer": "standard"
}
Please see the below sample which would fit your use case where I've created a custom analyzer which would fit your use case:
Sample Mapping:
PUT some_test_index
{
"settings": {
"analysis": {
"analyzer": {
"my_custom_analyzer": {
"type": "custom",
"tokenizer": "custom_tokenizer",
"filter": ["lowercase", "3_5_edge_ngram"]
}
},
"tokenizer": {
"custom_tokenizer": {
"type": "pattern",
"pattern": "\\w+_+[^a-zA-Z\\d\\s_]+|\\s+". <---- Note this pattern
}
},
"filter": {
"3_5_edge_ngram": {
"type": "edge_ngram",
"min_gram": 3,
"max_gram": 5
}
}
}
},
"mappings": {
"properties": {
"my_field":{
"type": "text",
"analyzer": "my_custom_analyzer"
}
}
}
}
The above mentioned pattern would simply ignore the tokens with the format like abc_$%^^##. As a result this token would not be indexed.
Note that the way the analyzer works is:
First executes tokenizer
Then applies the edge_ngram filter on the tokens generated.
You can verify by simply removing the edge_ngram filter in the above mapping to first understand what tokens are getting generated via Analyze API which would be as below:
POST some_test_index/_analyze
{
"analyzer": "my_custom_analyzer",
"text": "abc_name asda efg_!##!## 1213_adav"
}
Tokens generated:
{
"tokens" : [
{
"token" : "abc_name",
"start_offset" : 0,
"end_offset" : 8,
"type" : "word",
"position" : 0
},
{
"token" : "asda",
"start_offset" : 9,
"end_offset" : 13,
"type" : "word",
"position" : 1
},
{
"token" : "1213_adav",
"start_offset" : 25,
"end_offset" : 34,
"type" : "word",
"position" : 2
}
]
}
Note that the token efg_!##!## has been removed.
I've added edge_ngram fitler as you would want the search to be successful if you search with abc_ if your tokens generated via tokenizer is abc_name.
Sample Document:
POST some_test_index/_doc/1
{
"my_field": "abc_name asda efg_!##!## 1213_adav"
}
Query Request:
Use-case 1:
POST some_test_index/_search
{
"query": {
"match": {
"my_field": "abc_"
}
}
}
Use-case-2:
POST some_test_index/_search
{
"query": {
"match": {
"my_field": "efg_!##!##"
}
}
}
Responses:
Response for use-case-1:
{
"took" : 0,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 0.47992462,
"hits" : [
{
"_index" : "some_test_index",
"_type" : "_doc",
"_id" : "1",
"_score" : 0.47992462,
"_source" : {
"my_field" : "abc_name asda efg_!##!## 1213_adav"
}
}
]
}
}
Response for use-case-2:
{
"took" : 0,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 0,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
}
}
Updated Answer:
Create your mapping as follows based on the index I've created and let me know if that works:
PUT some_test_index
{
"settings": {
"analysis": {
"analyzer": {
"my_custom_analyzer": {
"type": "custom",
"tokenizer": "punctuation",
"filter": ["lowercase"]
}
},
"tokenizer": {
"punctuation": {
"type": "pattern",
"pattern": "\\w+_+[^a-zA-Z\\d\\s_]+|\\s+"
}
}
}
},
"mappings": {
"properties": {
"my_field":{
"type": "text",
"analyzer": "autocompete", <----- Assuming you have already this in setting
"search_analyzer": "my_custom_analyzer". <----- Note this
}
}
}
}
Please try and let me know if this works for all your use-cases.

Elasticsearch aggrecation give me 2 results insted of one result

I want to aggregate on the brand field and is give me two results instead of one
The brands_aggs give me from this text
{name : "Brand 1"}
2 results
Brand and 1
But Why I need only Brand 1
is separate the word brand and 1 from (Brand 1)
and is give me 2 results in the aggrecation
my mappings where I want to aggregate
mapping = {
"mappings": {
"product": {
"properties": {
"categories": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
},
"fielddata": True
}
"brand": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
},
"fielddata": True
}
}
}
}
}
my post request
{
"query" : {
"bool": {
"must": [
{"match": { "categories": "AV8KW5Wi31qHZdVeXG4G" }}
]
}
},
"size" : 0,
"aggs" : {
"brand_aggs" : {
"terms" : { "field" : "brand" }
},
"categories_aggs" : {
"terms" : { "field" : "categories" }
}
}
}
response from the server
{
"took": 18,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 0,
"hits": []
},
"aggregations": {
"categories_aggs": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "av8kw5wi31qhzdvexg4g",
"doc_count": 1
},
{
"key": "av8kw61c31qhzdvexg4h",
"doc_count": 1
},
{
"key": "av8kxtch31qhzdvexg4a",
"doc_count": 1
}
]
},
"brand_aggs": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "1", <==== I dont need this , why is give me that ??
"doc_count": 1
},
{
"key": "brand",
"doc_count": 1
}
]
},
}
}
Your mapping has property fields which is used when you want to have multiple analyzers for the same field. In your case valid name of your field is 'brand.keyword'. When you call your aggregate for just 'brand' it use default mapping defined for string.
So your query should be:
{
"query" : {
"bool": {
"must": [
{"match": { "categories": "AV8KW5Wi31qHZdVeXG4G" }}
]
}
},
"size" : 0,
"aggs" : {
"brand_aggs" : {
"terms" : { "field" : "brand.keyword" }
},
"categories_aggs" : {
"terms" : { "field" : "categories.keyword" }
}
}
}
Property field is useful when you want for example search the same property which multiple analyzers, for example:
"full_name": {
"type": "text",
"analyzer": "standard",
"boost": 1,
"fields": {
"autocomplete": {
"type": "text",
"analyzer": "ngram_analyzer"
},
"standard":{
"type": "text",
"analyzer": "standard"
}
}
},
You need to map your string as not_analyzed string, for that run the below query
PUT your_index/_mapping/your_type
{
"your_type": {
"properties": {
"brand": {
"type": "string",
"index": "analyzed",
"fields": {
"raw": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
}
Don't forget to replace the your_type and your_index with your type and index values.

Search query to retrieve nested documents in elasticsearch with _source disabled

I have the following mapping
{
"cloth": {
"dynamic" : false,
"_source" : {"enabled" : false },
"properties": {
"name": {
"type": "string",
"index": "analyzed"
},
"variation": {
"type": "nested",
"properties": {
"size": {
"type": "string",
"index": "not_analyzed"
},
"color": {
"type": "string",
"index": "not_analyzed"
}
}
}
}
}
}
I am not able to figure out a way to retrieve the nested object fields using the fields query.
{
"fields" : ["name" , "variation.size", "variation.color"],
"query" : {
"nested" : {
"path" : "variation",
"query" : {
"bool" : {
"must" : [
{ "term" : { "variation.size" : "XXL" } },
{ "term" : { "variation.color" : "red" } }
]
}
}
}
}
}
The above query returns
"_id" : "1",
"_score" : 1.987628,
"fields" : {
"variation.size" : [ "XXL", "XL" ],
"variation.color" : [ "red", "black" ],
"name" : [ "Test shirt" ]
}
When I tried
"fields" : ["name" , "variation"]
I got the error
status: 400
reason: "ElasticsearchIllegalArgumentException[field [variation] isn't a leaf field]"
Which is as expected.
How can I get the variation object as it is?
Expected Result. I need to retrieve the variable object as a whole so that I can preserve the association of size and color. Like "red" with "XXL".
"variation" : { "XXL" , "red"}
Update: Source is disabled for this Index Type.
If you use Source Filtering it will return the nested objects as a whole, your query would be:
{
"_source": [
"name",
"variation"
],
"query": {
"nested": {
"path": "variation",
"query": {
"bool": {
"must": [
{
"term": {
"variation.size": "XXL"
}
},
{
"term": {
"variation.color": "red"
}
}
]
}
}
}
}
}
You should use this:
"script_fields": {
"variation": {
"script": {
"inline": "doc['variation.size'].value + ' ' + doc['variation.red'].value"
}
}
}
I use elasticsearch v. 5.1.1

How to use facet filtering with nested documents on ElasticSearch

I have the following mapping:
curl -XPUT 'http://localhost:9200/bookstore/user/_mapping' -d '
{
"user": {
"properties": {
"user_id": { "type": "integer" },
"gender": { "type": "string", "index" : "not_analyzed" },
"age": { "type": "integer" },
"age_bracket": { "type": "string", "index" : "not_analyzed" },
"current_city": { "type": "string", "index" : "not_analyzed" },
"relationship_status": { "type": "string", "index" : "not_analyzed" },
"books" : {
"type": "nested",
"properties" : {
"b_oid": { "type": "string", "index" : "not_analyzed" },
"b_name": { "type": "string", "index" : "not_analyzed" },
"bc_id": { "type": "integer" },
"bc_name": { "type": "string", "index" : "not_analyzed" },
"bcl_name": { "type": "string", "index" : "not_analyzed" },
"b_id": { "type": "integer" }
}
}
}
}
}'
Now, I try to query for example for Users which have "gender": "Male", have bought book in a certain category "bcl_name": "Trivia" and show the "b_name" book titles. I somehow cannot get it to run.
I have the query
curl -XGET 'http://localhost:9200/bookstore/user/_search?pretty=1' -d '{
"size": 0,
"from": 0,
"query": {
"filtered": {
"query": {
"terms": {
"gender": [
"Male"
]
}
}
}
},
"facets": {
"CategoryFacet": {
"terms": {
"field": "books.b_name",
"size": 5,
"shard_size": 1000,
"order": "count"
},
"nested": "books",
"facet_filter": {
"terms": {
"books.bcl_name": [
"Trivia"
]
}
}
}
}
}'
which returns a result, but I'm not sure whether this is correct. I looked for some examples, and found this (http://www.spacevatican.org/2012/6/3/fun-with-elasticsearch-s-children-and-nested-documents/) for example. I'm able to rewrite my query like this:
curl -XGET 'http://localhost:9200/bookstore/user/_search?pretty=1' -d '{
"size": 0,
"from": 0,
"query": {
"filtered": {
"query": {
"terms": {
"gender": [
"Male"
]
}
},
"filter": {
"nested": {
"path": "books",
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"and": [
{
"term": {
"books.bcl_name": "Trivia"
}
}
]
}
}
}
}
}
}
},
"facets": {
"CategoryFacet": {
"terms": {
"field": "books.b_name",
"size": 5,
"shard_size": 1000,
"order": "count"
},
"nested": "books"
}
}
}'
which shows different results.
I, as a beginner, am a litte lost right now. Can someone please give me hint on how to solve this`? Thanks a lot in advance!
First query means:
Search for users whose gender : "Male"
But "CategoryFacet" includes the count of gender : "Male" AND
books.bcl_name : "Trivia"
So in result set you get all "Male" users, but your CategoryFacet gives you the count of "Male users AND whose books.bcl_name is Trivia".
In second query your "CategoryFacet" does not include extra filtering. It just returns the facets from the exact result set.

Resources