elasticsearch js api upsert with multiple fields match - node.js

I am using elasticsearch in a node environment.
This is my data model :
"_source": {
"repo_id": "ecom-test",
"pr_id": "1",
"pr_status": "approved",
"date_created": "2016-12-14T12:55:50.317Z",
"date_modified": "2016-12-14T12:55:50.317Z",
"comments": [
{
"tuple": {
"reviewee": "Max",
"reviewer": "Vic"
},
"nb": "1",
"type": "typo"
},
{
"tuple": {
"reviewee": "Antoine",
"reviewer": "Vic"
},
"nb": "2",
"type": "logic"
}
]
}
I add comments using the following code :
client.update({
index: 'loreal',
type: 'reviews',
id: reviewID,
body: {
script: {
inline: "ctx._source.comments.add(params.comment)",
lang: "painless",
params: {
comment: {
tuple: {
reviewee: data.reviewee,
reviewer: data.reviewer
},
nb: data.nb,
type: data.type
}
}
}
}
})
But when a comment with the same "reviewee", "reviewer" AND "logic" already exists i would like to update the "nb" attribute instead of creating a new item in the "comments" array.
I thought i would first make a search but I can't find a way to make a search that matches any element of the array "comments" that has those three values.
I really hope you guys can give me a hand with this, it's been a long time I am searching =s.
Thanks in advance

Given the struct of your document, I would have defined the comment field like a nested object: see the documentation example which is pretty close to your use case: https://www.elastic.co/guide/en/elasticsearch/guide/current/nested-objects.html#nested-objects
Considering that you got in param the reviewee value, I would use the combination of nested query and bool query. https://www.elastic.co/guide/en/elasticsearch/guide/2.x/nested-query.html
{
"query": {
"nested": {
"path": "comments",
"query": {
"bool": {
"must": [{
"match": {
"comments.tuple.reviewee": "your reviewee"
}
}, {
"match": {
"comments.tuple.reviewer": "your reviewee"
}
}, {
"term": {
"comments.type": "logic"
}
}]
}
}
}
}
Here the test I wrote:
PUT test
{
"mappings": {
"item": {
"properties": {
"comments": {
"type": "nested"
}
}
}
}
}
POST test/item
{ "comments": [
{
"tuple": {
"reviewee": "Max",
"reviewer": "Max"
},
"nb": "1",
"type": "logic"
},
{
"tuple": {
"reviewee": "Antoine",
"reviewer": "Vic"
},
"nb": "2",
"type": "logic"
}
]
}
POST test/item
{
"comments": [
{
"tuple": {
"reviewee": "Max",
"reviewer": "Max"
},
"nb": "1",
"type": "test"
},
{
"tuple": {
"reviewee": "Antoine",
"reviewer": "Vic"
},
"nb": "2",
"type": "logic"
}
]
}
POST test/item
{
"comments": [
{
"tuple": {
"reviewee": "Max",
"reviewer": "Mac"
},
"nb": "1",
"type": "logic"
},
{
"tuple": {
"reviewee": "Antoine",
"reviewer": "Vic"
},
"nb": "2",
"type": "logic"
}
]
}
GET /test/item/_search
{
"query": {
"nested": {
"path": "comments",
"query": {
"bool": {
"must": [{
"match": {
"comments.tuple.reviewee": "Max"
}
}, {
"match": {
"comments.tuple.reviewer": "Max"
}
}, {
"term": {
"comments.type": "logic"
}
}]
}
}
}
}}

Related

Filter data using nodejs and elasticsearch

I'm currently facing an issue with my datatable implemented in ReactJS. I'm retrieving data from elasticsearch and populating the datatable with it. The data retrieval process works fine without the filter applied, however, when I apply filters to the data, the datatable remains empty, even though the data _source has matching records.
The structure of the parameters I am sending is as follows:
{
pageIndex: 1,
pageSize: 10,
sort: { order: '', key: '' },
query: '',
filterData: {
analysis: [ '0', '1', '2', '3' ],
threat_level_id: [ '1', '2', '3', '4' ],
}
}
EndPoint:
POST /api/v1/events/public/list
Controller:
exports.getPublicEvents = async (req, res) => {
try {
client.ping()
const { pageIndex, pageSize, sort, query, filterData } = req.body
let esQuery = {
index: 'ns_*',
body: {
query: {
bool: {
must: [
{
match_all: {},
},
],
filter: [],
},
},
from: (pageIndex - 1) * pageSize,
size: pageSize,
},
}
if (query) {
esQuery.body.query.bool.must = [
{
match: {
'Event.info': {
query: query,
fuzziness: 'AUTO',
},
},
},
]
}
if (filterData.analysis.length > 0) {
esQuery.body.query.bool.filter.push({
terms: {
'Event.analysis': filterData.analysis,
},
})
}
if (filterData.threat_level_id.length > 0) {
esQuery.body.query.bool.filter.push({
terms: {
'Event.threat_level_id': filterData.threat_level_id,
},
})
}
let esResponse = await client.search(esQuery)
let data = esResponse.hits.hits.map((hit) => hit._source)
let total = esResponse.hits.total.value
res.status(200).json({
status: 'success',
data: data,
total: total,
})
} catch (error) {
res.status(500).json({
error: 'Error connecting to Elasticsearch',
errorMessage: error.message,
})
}
}
The controller below is without filters and it works just fine.
exports.getPublicEvents = async (req, res) => {
try {
client.ping()
const { pageIndex, pageSize, sort, query } = req.body
let esQuery = {
index: 'ns_*',
body: {
query: {
match_all: {},
},
from: (pageIndex - 1) * pageSize,
size: pageSize,
},
}
if (query) {
esQuery.body.query = {
match: {
'Event.info': {
query: query,
fuzziness: 'AUTO',
},
},
}
}
let esResponse = await client.search(esQuery)
let data = esResponse.hits.hits.map((hit) => hit._source)
let total = esResponse.hits.total.value
res.status(200).json({
status: 'success',
data: data,
total: total,
})
} catch (error) {
res.status(500).json({
error: 'Error connecting to Elasticsearch',
errorMessage: error.message,
})
}
}
ElasticSearech version: 7.17.8
Result of: console.log(JSON.stringify(esQuery))
{
"index": "INDEX_NAME",
"body": {
"query": {
"bool": {
"must": [{ "match_all": {} }],
"filter": [
{ "terms": { "Event.analysis": ["0", "1", "2"] } },
{ "terms": { "Event.threat_level_id": ["1", "2", "3", "4"] } }
]
}
},
"from": 0,
"size": 10
}
}
Data in elascticsearch schema
{
"#version": "1",
"#timestamp": "2023-02-01T14:43:09.997Z",
"Event": {
"info": ".......................",
"description": ".......................",
"analysis": 0,
"threat_level_id": "4",
"created_at": 1516566351,
"uuid": "5a64f74f0e543738c12bc973322",
"updated_at": 1675262417
}
}
Index Mapping
{
"index_patterns": ["INDEX_NAME"],
"template": "TEMPLATE_NAME",
"settings": {
"number_of_replicas": 0,
"index.mapping.nested_objects.limit": 10000000
},
"mappings": {
"dynamic": false,
"properties": {
"#timestamp": {
"type": "date"
},
"Event": {
"type": "nested",
"properties": {
"date_occured": {
"type": "date"
},
"threat_level_id": {
"type": "integer"
},
"description": {
"type": "text"
},
"is_shared": {
"type": "boolean"
},
"analysis": {
"type": "integer"
},
"uuid": {
"type": "text"
},
"created_at": {
"type": "date"
},
"info": {
"type": "text"
},
"shared_with": {
"type": "nested",
"properties": {
"_id": {
"type": "text"
}
}
},
"updated_at": {
"type": "date"
},
"author": {
"type": "text"
},
"Attributes": {
"type": "nested",
"properties": {
"data": {
"type": "text"
},
"type": {
"type": "text"
},
"uuid": {
"type": "text"
},
"comment": {
"type": "text"
},
"category": {
"type": "text"
},
"value": {
"type": "text"
},
"timestamp": {
"type": "date"
}
}
},
"organisation": {
"type": "nested",
"properties": {
"name": {
"type": "text"
},
"uuid": {
"type": "text"
}
}
},
"Tags": {
"type": "nested",
"properties": {
"color": {
"type": "text"
},
"name": {
"type": "text"
}
}
},
"TLP": {
"type": "nested",
"properties": {
"color": {
"type": "text"
},
"name": {
"type": "text"
}
}
}
}
}
}
}
}
Event is a nested field, so you need to use nested queries, like this:
{
"index": "INDEX_NAME",
"body": {
"query": {
"bool": {
"must": [{ "match_all": {} }],
"filter": [
{
"nested": {
"path": "Event",
"query": {"terms": { "Event.analysis": ["0", "1", "2"] }}
}
},
{
"nested": {
"path": "Event",
"query": {"terms": { "Event.threat_level_id": ["1", "2", "3", "4"] }}
}
}
]
}
},
"from": 0,
"size": 10
}
}

How to use mongo Query to fetch data from nested documet

Here is the document.
How to use mongoQuery on branchId which have to value is e63a5343-871c-47ee-8fe6-31d9b7f9fa1b1 and returns data only have this branchId.
I'm using
db.collection.find({
'checkinDetails.presenceTable': { $elemMatch:
{'priorityDevice.branchId':"e63a5343-871c-47ee-8fe6-31d9b7f9fa1b1"}}
{
"checkinDetails": {
"userCount": NumberInt("9"),
"presenceTable": [
{
"entity": {
"entityCategory": "Users",
"entityType": "User",
"entityName": "Raghvendra",
"id": "25d156df-2c7e-4506-a5b0-969bf5d0745b"
},
"fromState": "current",
"priorityDevice": {
"aclFlag": false,
"zoneGranularity": NumberInt("300000"),
"csId": "0577936e-f81c-4386-bae7-0255a28ee374",
"branchId": "e63a5343-871c-47ee-8fe6-31d9b7f9fa1b1",
"txPower": NumberInt("-54"),
"filterRSSI": -63.15999016102085,
"rssi": NumberInt("-55"),
"time": "2019-01-17T14:38:11.804Z",
"zoneId": "e813b2f3-88f8-4111-b2bb-50bb7dfb0ba7",
"deviceId": "26341b09-9b16-4de4-a9ce-74dd3a858be8"
}
},
{
"entity": {
"entityCategory": "Users",
"entityType": "User",
"entityName": "Rahul Saini",
"id": "8f067f5b-6a8f-4237-a760-109303def580"
},
"fromState": "current",
"priorityDevice": {
"aclFlag": false,
"zoneGranularity": NumberInt("300000"),
"csId": "0577936e-f81c-4386-bae7-0255a28ee374",
"branchId": "e63a5343-871c-47ee-8fe6-31d9b7f9fa1b",
"txPower": NumberInt("-54"),
"filterRSSI": -66.44148222664761,
"rssi": NumberInt("-61"),
"time": "2019-01-17T14:38:11.803Z",
"zoneId": "e813b2f3-88f8-4111-b2bb-50bb7dfb0ba7",
"deviceId": "26341b09-9b16-4de4-a9ce-74dd3a858be8"
}
},
{
"entity": {
"entityCategory": "Users",
"entityType": "User",
"entityName": "Shubham Saurabh",
"id": "4f0fe333-3f9f-457f-8046-5476573323fb"
},
"fromState": "current",
"priorityDevice": {
"aclFlag": false,
"zoneGranularity": NumberInt("300000"),
"csId": "0577936e-f81c-4386-bae7-0255a28ee374",
"branchId": "e63a5343-871c-47ee-8fe6-31d9b7f9fa1b1",
"txPower": NumberInt("-62"),
"filterRSSI": -61.489655691672645,
"rssi": NumberInt("-57"),
"time": "2019-01-17T14:38:11.799Z",
"zoneId": "e813b2f3-88f8-4111-b2bb-50bb7dfb0ba7",
"deviceId": "26341b09-9b16-4de4-a9ce-74dd3a858be8"
}
}
]
"type": "presenceTable"
}
Regarding to your comment. Try this example to fetch data using MongoDB query.
db.collection.find({ checkinDetails: { presenceTable: { priorityDevice : { branchId :"e63a5343-871c-47ee-8fe6-31d9b7f9fa1b1"} } } })
More infromation you can find here : https://docs.mongodb.com/manual/tutorial/query-embedded-documents/

How to check if all values exists inside nested object elastic search

I have following document in ES :
[
{
"event_id": 123,
"event_name": "test event",
"event_date": "2018-12-21",
"ticket_group": [
{
"available": 8,
"price": 8,
"id": "159831",
"parking_passes_available": 0,
"field_values": [
{
"field_id": 589,
"field_value": "KUMAR"
},
{
"field_id": 717,
"field_value": "AMIT"
},
{
"field_id": 1360,
"field_value": "SAM"
},
{
"field_id": 2239,
"field_value": ""
},
{
"field_id": 2240,
"field_value": ""
},
{
"field_id": 2241,
"field_value": ""
},
{
"field_id": 2242,
"field_value": ""
}
]
}
]
}
]
and i want to search with multiple field_id and field_value with AND operator. But it works if there is single condition but not for multiple cases. Here is what i have tried so far :
{
"query": {
"bool": {
"must": [
{
"nested": {
"path": "ticket_group",
"score_mode": "max",
"inner_hits": {
"from": 0,
"size": 10000
},
"query": {
"bool": {
"must": [
{
"nested": {
"path": "ticket_group.field_values",
"score_mode": "max",
"inner_hits": {
"from": 0,
"size": 10000
},
"query": {
"bool": {
"must": [
{
"bool": {
"must": [
{
"match": {
"ticket_group.field_values.field_id": 589
}
},
{
"match": {
"ticket_group.field_values.field_value": "KUMAR"
}
}
]
}
},
{
"bool": {
"must": [
{
"match": {
"ticket_group.field_values.field_id": 717
}
},
{
"match": {
"ticket_group.field_values.field_value": "AMIT"
}
}
]
}
}
]
}
}
}
}
]
}
}
}
}
]
}
},
"size": 10,
"from": 0,
"sort": {
"event_date": {
"order": "asc"
}
}
}
i want to search ticket group if field_id=717 with value "amit" and field_id=589 with value "kumar" exists in field_values object inside ticket_group. Using above query i am getting no records while objects with both values exist in field_values.
Can anyone help to build a such query ?
Thank You
Below is what you are looking for. You simply need to push the second level nested into two must clauses.
POST <your_index_name>/_search
{
"query":{
"bool":{
"must":[
{
"nested":{
"path":"ticket_group",
"score_mode":"max",
"inner_hits":{
},
"query":{
"bool":{
"must":[
{
"nested":{
"path":"ticket_group.field_values",
"score_mode":"max",
"inner_hits":{
"name":"inner_clause_1"
},
"query":{
"bool":{
"must":[
{
"match":{
"ticket_group.field_values.field_id":589
}
},
{
"match":{
"ticket_group.field_values.field_value":"KUMAR"
}
}
]
}
}
}
},
{
"nested":{
"path":"ticket_group.field_values",
"score_mode":"max",
"inner_hits":{
"name":"inner_clause_2"
},
"query":{
"bool":{
"must":[
{
"match":{
"ticket_group.field_values.field_id":717
}
},
{
"match":{
"ticket_group.field_values.field_value":"AMIT"
}
}
]
}
}
}
}
]
}
}
}
}
]
}
}
}
Notice that I've named the inner_hits in the second level nested queries.
If you don't do that(try by removing the name key in the inner_hits), then you would only see the inner_hit for the last clause which ends up overwriting the inner_hits result of the first nested clause.
Hope this helps!

Elastic Search: Matching fields in different nested objects

I am new to Elastic Search and this is my user index:
{
"user": {
"properties": {
"branches": {
"type": "nested"
},
"lists": {
"type": "nested"
},
"events": {
"type": "nested"
},
"optOuts": {
"type": "nested"
}
}
}
}
Here, branches, events and lists will contain the field id(int),countryIso(String)..
I need to find users having emails who belong to countryIso 'XX' for example.
{
"query": {
"bool": {
"must": [
{
"exists": {
"field": "email"
}
},
{
"match": {
"prog_id": 3
}
},
{
"nested": {
"path": [
"branches"
],
"query": {
"query_string": {
"fields": [
"branches.countryIso"
],
"query": "AE KW"
}
}
}
}
]
}
}
}
This way I can get them if they have that country in the branches object. What I want is that the countryIso is there in the branches or lists or events.
Note: any of these might be empty i.e. branches may not be there or lists miht not be there etc. Or lists might be there with no countryIso..
I tried this:
{
"query": {
"bool": {
"must": [
{
"exists": {
"field": "email"
}
},
{
"match": {
"prog_id": 3
}
},
{
"nested": {
"path": [
"branches"
],
"query": {
"query_string": {
"fields": [
"branches.countryIso"
],
"query": "AE KW"
}
}
}
},
{
"nested": {
"path": [
"lists"
],
"query": {
"query_string": {
"fields": [
"lists.countryIso"
],
"query": "AE KW"
}
}
}
}
]
}
}
}
AND
{
"query": {
"bool": {
"must": [
{
"exists": {
"field": "email"
}
},
{
"match": {
"prog_id": 3
}
},
{
"nested": {
"path": [
"branches",
"lists"
],
"query": {
"query_string": {
"fields": [
"branches.countryIso",
"lists.countryIso"
],
"query": "AE KW"
}
}
}
}
]
}
}
}
But neither works.

ElasticSearch query stops working with big amount of data

The problem: I have 2 identical in terms of settings and mappings indexes.
The first index contains only 1 document.
The second index contains the same document + 16M of others.
When I'm running the query on the first index it returns the document, but when I do the same query on the second — I receive nothing.
Indexes settings:
{
"tasks_test": {
"settings": {
"index": {
"analysis": {
"analyzer": {
"tag_analyzer": {
"filter": [
"lowercase",
"tag_filter"
],
"tokenizer": "whitespace",
"type": "custom"
}
},
"filter": {
"tag_filter": {
"type": "word_delimiter",
"type_table": "# => ALPHA"
}
}
},
"creation_date": "1444127141035",
"number_of_replicas": "2",
"number_of_shards": "5",
"uuid": "wTe6WVtLRTq0XwmaLb7BLg",
"version": {
"created": "1050199"
}
}
}
}
}
Mappings:
{
"tasks_test": {
"mappings": {
"Task": {
"dynamic": "false",
"properties": {
"format": "dateOptionalTime",
"include_in_all": false,
"type": "date"
},
"is_private": {
"type": "boolean"
},
"last_timestamp": {
"type": "integer"
},
"name": {
"analyzer": "tag_analyzer",
"type": "string"
},
"project_id": {
"include_in_all": false,
"type": "integer"
},
"user_id": {
"include_in_all": false,
"type": "integer"
}
}
}
}
}
The document:
{
"_index": "tasks_test",
"_type": "Task",
"_id": "1",
"_source": {
"is_private": false,
"name": "135548- test with number",
"project_id": 2,
"user_id": 1
}
}
The query:
{
"query": {
"filtered": {
"query": {
"bool": {
"must": [
[
{
"match": {
"_all": {
"query": "135548",
"type": "phrase_prefix"
}
}
}
]
]
}
},
"filter": {
"bool": {
"must": [
{
"term": {
"is_private": false
}
},
{
"terms": {
"project_id": [
2
]
}
},
{
"terms": {
"user_id": [
1
]
}
}
]
}
}
}
}
}
Also, some findings:
if I replace _all with name everything works
if I replace match_phrase_prefix with match_phrase works too
ES version: 1.5.1
So, the question is: how to make the query work for the second index without mentioned hacks?

Resources