I have a nested object as follows
{"stats": {"abc": {"NumSources": 6, "Size": 12754890006, "Sources": {"NodeA": {"NumFiles": 246, "Size": 509071269}, "NodeB": {"NumFiles": 104, "Size": 823385346}, "NodeC": {"NumFiles": 466, "Size": 1259819487}, "NodeD": {"NumFiles": 178, "Size": 1712383515}, "NodeE-daemon": {"NumFiles": 79, "Size": 7338}, "NodeF": {"NumFiles": 244, "Size": 8450223051}}}}}
and here is the mapping i create
put statsview
{
"mappings":{
"test2":{
"properties":{
"stats":{
"type":"nested",
"properties":{
"abc":{
"type":"nested",
"include_in_parent": true,
"properties": {
"NumSources": {
"type": "long"
},
"Size": {
"type": "long"
},
"Sources": {
"type":"nested",
"include_in_parent": true,
"properties": {
"NodeA": {
"type":"nested",
"include_in_parent": true,
"properties": {
"NumFiles": {
"type": "long"
},
"Size": {
"type": "long"
}
}
},
"NodeB": {
"type":"nested",
"include_in_parent": true,
"properties": {
"NumFiles": {
"type": "long"
},
"Size": {
"type": "long"
}
}
},
"NodeC": {
"type":"nested",
"include_in_parent": true,
"properties": {
"NumFiles": {
"type": "long"
},
"Size": {
"type": "long"
}
}
},
"NodeD": {
"type":"nested",
"include_in_parent": true,
"properties": {
"NumFiles": {
"type": "long"
},
"Size": {
"type": "long"
}
}
},
"NodeE": {
"type":"nested",
"include_in_parent": true,
"properties": {
"NumFiles": {
"type": "long"
},
"Size": {
"type": "long"
}
}
},
"NodeF": {
"type":"nested",
"include_in_parent": true,
"properties": {
"NumFiles": {
"type": "long"
},
"Size": {
"type": "long"
}
}
}
}
}
}
}
}
}
}
}
}
}
Note that, at each level i am including the spec "include_in_parent": true,
now, in my Kibana Discover view, i see just one record, which makes sense.
GET statsview/_search
{
"query": {
"match_all": {}
}
}
which gives me
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 1,
"hits": [
{
"_index": "statsview",
"_type": "test2",
"_id": "1",
"_score": 1,
"_source": {
"logs": {
"abc": {
"NumSources": 6,
"Size": 12754890006,
"Sources": {
"NodeA": {
"NumFiles": 246,
"Size": 509071269
},
"NodeB": {
"NumFiles": 104,
"Size": 823385346
},
"NodeC": {
"NumFiles": 466,
"Size": 1259819487
},
"NodeD": {
"NumFiles": 178,
"Size": 1712383515
},
"NodeE-daemon": {
"NumFiles": 79,
"Size": 7338
},
"NodeF": {
"NumFiles": 244,
"Size": 8450223051
}
}
}
}
}
}
]
}
}
So i see this record in Kibana's Discover view. But i cant run queries like
logs.abc.Sources
so i want to see all of the sources i.e NodeA to NodeF. I even tried
logs.abc.Sources:[NodeA To NodeF]
but i still get status 200.
Is there support in Kibana for running such queries for nested objects?
How can i visualize these nested objects?
Is my mapping correct?
Related
Hey I have a problem with my mongoDb search. I need to add one more index to my search for better result. But since its not in mappings whenever i perform my search as following it returns me nothing.
`
embeddedDocument: {
path: 'produces',
operator: {
compound: {
"must": [
{
autocomplete: {
"path": "produces.name",
"query": val,
"fuzzy": {
"maxEdits": 2,
"prefixLength": 3
}
}
},
{
equals: { path: 'produces.deleted', value: false }
}
],
}
}
}
and here is my index definition
{
"mappings": {
"dynamic": false,
"fields": {
"companyName": [
{
"analyzer": "lucene.standard",
"type": "string"
},
{
"foldDiacritics": true,
"maxGrams": 6,
"minGrams": 2,
"tokenization": "edgeGram",
"type": "autocomplete"
}
],
"produces": {
"type": "embeddedDocuments",
"fields": {
"name": [
{
"analyzer": "lucene.standard",
"type": "string"
},
{
"foldDiacritics": true,
"maxGrams": 6,
"minGrams": 2,
"tokenization": "edgeGram",
"type": "autocomplete"
}
]
}
},
"status": {
"type": "string"
}
}
}
}
So i need to update my index definition as following.
{
"mappings": {
"dynamic": false,
"fields": {
"companyName": [
{
"analyzer": "lucene.standard",
"type": "string"
},
{
"foldDiacritics": true,
"maxGrams": 6,
"minGrams": 2,
"tokenization": "edgeGram",
"type": "autocomplete"
}
],
"produces": {
"type": "embeddedDocuments",
"fields": {
"deleted": {
"type": "boolean"
},
"name": [
{
"analyzer": "lucene.standard",
"type": "string"
},
{
"foldDiacritics": true,
"maxGrams": 6,
"minGrams": 2,
"tokenization": "edgeGram",
"type": "autocomplete"
}
]
}
},
"status": {
"type": "string"
}
}
}
}
`
But my problem is i dont know where to add this index definition on the mongoDB side so i can filter with deleted false criteria.
I tried to add index to mongoDB to produces collection but it doesnt work and i still get empty result.
Is there any way to fetch documents with pagination using the driver's built-in mapper? The findAll() method accepts pageState as an option but does not return it to use it in next calls.
const result = await this.repository.findAll({ limit: 1 }, { fetchSize: 1 });
console.log(JSON.stringify(result));
returns this JSON:
{
"_rs": {
"info": {
"queriedHost": "127.0.0.1:9042",
"triedHosts": {
"127.0.0.1:9042": null
},
"speculativeExecutions": 0,
"achievedConsistency": 10,
"isSchemaInAgreement": true
},
"rows": [],
"rowLength": 0,
"columns": [
{
"name": "namespace_id",
"type": {
"code": 13,
"type": null
}
},
{
"name": "archived_at",
"type": {
"code": 15,
"type": null
}
},
{
"name": "channel_id",
"type": {
"code": 13,
"type": null
}
},
{
"name": "created_at",
"type": {
"code": 11,
"type": null
}
},
{
"name": "department_id",
"type": {
"code": 13,
"type": null
}
},
{
"name": "id",
"type": {
"code": 13,
"type": null
}
},
{
"name": "identifier_type",
"type": {
"code": 13,
"type": null
}
},
{
"name": "identifier_value",
"type": {
"code": 13,
"type": null
}
},
{
"name": "last_assignee_id",
"type": {
"code": 9,
"type": null
}
}
],
"pageState": null
},
"_info": {
"keyspace": "omnichannel",
"tables": [
{
"name": "ticket_by_archive_time",
"isView": false
},
{
"name": "ticket_by_id",
"isView": false
},
{
"name": "ticket_by_identifier",
"isView": false
}
],
"_mappings": {},
"_columns": {},
"_documentProperties": {}
},
"_isEmptyLwt": false,
"length": 0,
"pageState": null
}
The problem was passing limit option, removing it and using only fetchSize fixed the issue.
I am creating a filters application for products, and i am using nodejs and elasticsearch for filtering products. I have stuck on price range filter, it shows wrong options values.
Here is mapping.
{
"elasticsearch": {
"default_mapping": {
"properties": {
"variants.option_values": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"completion": {
"type": "completion",
"analyzer": "standard"
}
}
},
"variants.price": {
"type": "float",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"text": {
"type": "text"
}
}
}
}
}
}
}
My Query
{
"_source": [
"variants.option_values",
"variants.price"
],
"query": {
"bool": {
"filter": {
"range": {
"variants.price": {
"gte": 25000,
"lte": 30000
}
}
}
}
},
"aggs": {
"sizes": {
"terms": {
"field": "variants.option_values.keyword",
"order": {
"_count": "desc"
},
"size": 750
}
}
}
}
And Elasticsearch's Result
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 4.8374715,
"hits": [
{
"_index": "my-products",
"_type": "products",
"_id": "4680333656118",
"_score": 4.8374715,
"_ignored": [
"product_type.completion"
],
"_source": {
"variants": [
{
"price": "30000.00",
"option_values": [
"Small"
]
},
{
"price": "20000.00",
"option_values": [
"Medium"
]
},
{
"price": "25000.00",
"option_values": [
"Large"
]
}
]
}
}
]
},
"aggregations": {
"sizes": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "Large",
"doc_count": 1
},
{
"key": "Medium",
"doc_count": 1
},
{
"key": "Small",
"doc_count": 1
}
]
}
}
}
I am expecting an aggregations's buckets must have Size Large and Small. And must not include Medium. Because Size Medium's price in less than the filtered price.(In other words size medium is not available in filtered price).
Here is my mapping
"elasticsearch": {
"default_mapping": {
"properties": {
"placement": {
"type": "nested"
},
"collects": {
"properties": {
"collection_id": {
"type": "long",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"text": {
"type": "text"
}
}
},
"id": {
"type": "long",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"text": {
"type": "text"
}
}
},
"product_id": {
"type": "long",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"text": {
"type": "text"
}
}
}
}
},
"variants.percent_sale": {
"type": "float",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"text": {
"type": "text"
}
}
},
"variants.option_values": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"completion": {
"type": "completion",
"analyzer": "standard"
}
}
},
"variants.price": {
"type": "float",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"text": {
"type": "text"
}
}
},
"variants.compare_at_price": {
"type": "float",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"text": {
"type": "text"
}
}
},
"variants.weight": {
"type": "float",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"text": {
"type": "text"
}
}
},
"variants.sku": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"completion": {
"type": "completion",
"analyzer": "standard"
}
}
},
"tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"completion": {
"type": "completion",
"analyzer": "standard"
}
}
},
"vendor": {
"type": "text",
"analyzer": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"completion": {
"type": "completion",
"analyzer": "standard"
}
}
},
"product_type": {
"type": "text",
"analyzer": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"completion": {
"type": "completion",
"analyzer": "standard"
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"completion": {
"type": "completion",
"analyzer": "standard"
}
}
},
"options.values": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"completion": {
"type": "completion",
"analyzer": "standard"
}
}
},
"articles.title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"completion": {
"type": "completion",
"analyzer": "standard"
}
}
},
"articles.tags": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
},
"completion": {
"type": "completion",
"analyzer": "standard"
}
}
}
}
}
}
Mapping issue: Resolved when i set variants as nested type
inner_hits done the great job!
Here is results
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 0,
"hits": [
{
"_index": "my-products",
"_type": "products",
"_id": "5012966080571",
"_score": 0,
"_source": {
"variants": [
{
"price": "30000.00"
},
{
"price": "20000.00"
},
{
"price": "25000.00"
}
]
},
"inner_hits": {
"variants": {
"hits": {
"total": {
"value": 2,
"relation": "eq"
},
"max_score": null,
"hits": [
{
"_index": "my-products",
"_type": "products",
"_id": "5012966080571",
"_nested": {
"field": "variants",
"offset": 0
},
"_score": null,
"_source": {
"id": 34120727560251,
"product_id": 5012966080571,
"title": "Small",
"price": "30000.00",
"sku": "super-1101",
"position": 1,
"inventory_policy": "deny",
"compare_at_price": "30500.00",
"fulfillment_service": "manual",
"inventory_management": "shopify",
"option1": "Small",
"option2": null,
"option3": null,
"created_at": "2020-08-11T11:09:29-04:00",
"updated_at": "2020-08-11T11:09:49-04:00",
"taxable": true,
"barcode": "",
"grams": 144,
"image_id": null,
"weight": 144,
"weight_unit": "g",
"inventory_item_id": 35675740274747,
"inventory_quantity": 2,
"old_inventory_quantity": 2,
"requires_shipping": true,
"admin_graphql_api_id": "gid://shopify/ProductVariant/34120727560251",
"option_values": [
"Small"
],
"percent_sale": 1
},
"sort": [
30000
]
},
{
"_index": "my-products",
"_type": "products",
"_id": "5012966080571",
"_nested": {
"field": "variants",
"offset": 2
},
"_score": null,
"_source": {
"id": 34120727625787,
"product_id": 5012966080571,
"title": "Large",
"price": "25000.00",
"sku": "super-1103",
"position": 3,
"inventory_policy": "deny",
"compare_at_price": "25500.00",
"fulfillment_service": "manual",
"inventory_management": "shopify",
"option1": "Large",
"option2": null,
"option3": null,
"created_at": "2020-08-11T11:09:29-04:00",
"updated_at": "2020-08-11T11:10:03-04:00",
"taxable": true,
"barcode": "",
"grams": 144,
"image_id": null,
"weight": 144,
"weight_unit": "g",
"inventory_item_id": 35675740340283,
"inventory_quantity": 4,
"old_inventory_quantity": 4,
"requires_shipping": true,
"admin_graphql_api_id": "gid://shopify/ProductVariant/34120727625787",
"option_values": [
"Large"
],
"percent_sale": 1
},
"sort": [
25000
]
}
]
}
}
}
}
]
},
"aggregations": {
"variants_options": {
"doc_count": 3,
"inner": {
"doc_count": 2,
"sizes": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "Large",
"doc_count": 1
},
{
"key": "Small",
"doc_count": 1
}
]
}
}
}
}
}
Thanks Gibbs
I am using elasticsearch with mongoosastic npm module. I am trying to apply filter on geo coordinates having following model structure
geoLocation: {
type: {
type: String,
default: 'Point'
},
coordinates: [Number] //orders should be lat,lng
}
with the mapping as follows
{
"events": {
"settings": {
"analysis": {
"filter": {
"edgeNGram_filter": {
"type": "edgeNGram",
"min_gram": 1,
"max_gram": 50,
"side": "front"
}
},
"analyzer": {
"edge_nGram_analyzer": {
"type": "custom",
"tokenizer": "edge_ngram_tokenizer",
"filter": [
"lowercase",
"asciifolding",
"edgeNGram_filter"
]
},
"whitespace_analyzer": {
"type": "custom",
"tokenizer": "whitespace",
"filter": [
"lowercase",
"asciifolding"
]
}
},
"tokenizer": {
"edge_ngram_tokenizer": {
"type": "edgeNGram",
"min_gram": "1",
"max_gram": "50",
"token_chars": [
"letter",
"digit"
]
}
}
}
},
"mappings": {
"event": {
"_all": {
"index_analyzer": "nGram_analyzer",
"search_analyzer": "whitespace_analyzer"
},
"properties": {
"title": {
"type": "string",
"index": "not_analyzed"
},
"geoLocation": {
"index": "not_analyzed",
"type": "geo_point"
}
}
}
}
}
}
Query
{
"query": {
"multi_match": {
"query": "the",
"fields": ["title", ]
}
},
"filter" : {
"geo_distance" : {
"distance" : "200km",
"geoLocation.coordinates" : {
"lat" : 19.007452,
"lon" : 72.831556
}
}
}
}
I am unable to create indexing on the geo coordinates with the following model structure, I dont understand if it is not possible to index geo coordinates with the above model structure because in my case the coordinates has order as lat,long and I have found somewhere that elasticsearch expects coordinates order as long,lat.
Error
Error: SearchPhaseExecutionException[Failed to execute phase [query],
all shards failed; shardFailures {[CDHdgtJnTbeu8tl2mDfllg][events][0]:
SearchParseException[[events][0]: from[-1],size[-1]: Parse Failure
[Failed to parse source
curl -XGET localhost:9200/events
{
"events": {
"aliases": {},
"mappings": {
"1": {
"properties": {
"location": {
"type": "double"
},
"text": {
"type": "string"
}
}
},
"event": {
"properties": {
"city": {
"type": "string"
},
"endTime": {
"type": "date",
"format": "dateOptionalTime"
},
"geo_with_lat_lon": {
"type": "geo_point",
"lat_lon": true
},
"isActive": {
"type": "boolean"
},
"isRecommended": {
"type": "boolean"
},
"location": {
"type": "string"
},
"title": {
"type": "string"
}
}
}
},
"settings": {
"index": {
"creation_date": "1461675012489",
"uuid": "FT-xVUdPQtyuKFm4J4Rd7g",
"number_of_replicas": "1",
"number_of_shards": "5",
"events": {
"mappings": {
"event": {
"_all": {
"enabled": "false",
"search_analyzer": "whitespace_analyzer",
"index_analyzer": "nGram_analyzer"
},
"properties": {
"geoLocation": {
"coordinates": {
"type": "geo_shape",
"index": "not_analyzed"
}
},
"location": {
"type": "string",
"index": "not_analyzed"
},
"title": {
"type": "string",
"index": "not_analyzed"
},
"geo_with_lat_lon": {
"type": "geo_point",
"lat_lon": "true",
"index": "not_analyzed"
}
}
}
},
"settings": {
"analysis": {
"analyzer": {
"edge_nGram_analyzer": {
"type": "custom",
"filter": [
"lowercase",
"asciifolding",
"edgeNGram_filter"
],
"tokenizer": "edge_ngram_tokenizer"
},
"whitespace_analyzer": {
"type": "custom",
"filter": [
"lowercase",
"asciifolding"
],
"tokenizer": "whitespace"
}
},
"filter": {
"edgeNGram_filter": {
"max_gram": "50",
"type": "edgeNGram",
"min_gram": "1",
"side": "front"
}
},
"tokenizer": {
"edge_ngram_tokenizer": {
"max_gram": "50",
"type": "edgeNGram",
"min_gram": "1",
"token_chars": [
"letter",
"digit"
]
}
}
}
}
},
"version": {
"created": "1070099"
}
}
},
"warmers": {}
}
}
I got a solution for my question
Mapping
PUT /geo_test
{
"mappings": {
"type_test": {
"properties": {
"name": {
"type": "string"
},
"geoLocation": {
"type": "nested",
"properties": {
"coordinates": {
"type": "geo_point",
"lat_lon": true
}
}
}
}
}
}
}
Query
POST /geo_test/type_test/_search
{
"query": {
"filtered": {
"filter": {
"nested": {
"path": "geoLocation",
"query": {
"filtered": {
"filter": {
"geo_distance": {
"distance": 5,
"distance_unit": "km",
"geoLocation.coordinates": {
"lat": 41.12,
"lon": -71.34
}
}
}
}
}
}
}
}
}
}
As shown below, there are two types in my city index - zoo and hotel. How do I find all zoos having a hotel in 1KM radius? Here is the mapping of my index :
GET /city/_mapping
{
"city": {
"mappings": {
"hotel": {
"properties": {
"location": {
"type": "geo_point"
},
"name": {
"type": "string"
}
}
},
"zoo": {
"properties": {
"location": {
"type": "geo_point"
},
"name": {
"type": "string"
}
}
}
}
}
}
You can do it with a geo-distance filter for the whole index (just don't specify a type).
As I quick test I created an index like this:
PUT /test_index/
{
"mappings": {
"hotel": {
"properties": {
"location": {
"type": "geo_point"
},
"name": {
"type": "string"
}
}
},
"zoo": {
"properties": {
"location": {
"type": "geo_point"
},
"name": {
"type": "string"
}
}
}
}
}
Added a couple of documents
POST /test_index/_bulk
{"index":{"_type":"hotel","_id":1}}
{"name":"hotel1","location":{"lat" : 40.001, "lon" : -70.001}}
{"index":{"_type":"zoo","_id":1}}
{"name":"zoo1","location":{"lat" : 40.002, "lon" : -70.002}}
And then I can search like this. This query returns the one document:
POST /test_index/_search
{
"query": {
"filtered": {
"filter": {
"geo_distance": {
"distance": 200,
"distance_unit": "km",
"location": {
"lat": 40,
"lon": -70
}
}
}
}
}
}
...
{
"took": 2,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 1,
"max_score": 1,
"hits": [
{
"_index": "test_index",
"_type": "hotel",
"_id": "1",
"_score": 1,
"_source": {
"name": "hotel1",
"location": {
"lat": 40.001,
"lon": -70.001
}
}
}
]
}
}
And this query returns both:
POST /test_index/_search
{
"query": {
"filtered": {
"filter": {
"geo_distance": {
"distance": 300,
"distance_unit": "km",
"location": {
"lat": 40,
"lon": -70
}
}
}
}
}
}
...
{
"took": 5,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 2,
"max_score": 1,
"hits": [
{
"_index": "test_index",
"_type": "hotel",
"_id": "1",
"_score": 1,
"_source": {
"name": "hotel1",
"location": {
"lat": 40.001,
"lon": -70.001
}
}
},
{
"_index": "test_index",
"_type": "zoo",
"_id": "1",
"_score": 1,
"_source": {
"name": "zoo1",
"location": {
"lat": 40.002,
"lon": -70.002
}
}
}
]
}
}
Here's the code I used to test it:
http://sense.qbox.io/gist/948d23a5327cf5f22dd368146f37d09e30765fee