I'm getting started with ElasticSearch and I'm getting into troubles (I'm not understanding as It has to be) how to search.
First, I have this two documents:
{
"took": 133
"timed_out": false
"_shards": {
"total": 5
"successful": 5
"failed": 0
}
"hits": {
"total": 2
"max_score": 1
"hits": [2]
0: {
"_index": "app"
"_type": "player"
"_id": "AVcLCOgAi_gt2Fih02MK"
"_score": 1
"_source": {
"nickName": "sarasa"
"birthDate": "1994-11-05T13:15:30.000Z"
"state": "sarasa"
"adminState": "sarasa"
"id": ""
"account": {
"type": "yojuego"
"id": "asasdfa"
"password": "asd fasd"
}
}
}
1: {
"_index": "app"
"_type": "player"
"_id": "AVcQ7JNVi_gt2Fih02MN"
"_score": 1
"_source": {
"nickName": "facundo"
"birthDate": "1994-11-05T13:15:30.000Z"
"state": "verdura"
"adminState": "sudo"
"id": ""
"account": {
"type": "yojuego"
"id": "facundo#facundo"
"password": "pepe"
}
}
}
}
}
}
I want to get where account.id = "facundo#facundo" and account.type = "yojuego".
I'm doing this:
client.search({
index: 'app',
type: 'player',
query: {
bool: {
must: [
{ term: { "account.id": 'facundo#facundo' } },
{ term: { "account.type": 'yojuego' } }
],
}
}
}, (error, response, status) => {
if (error) {
res.json(400, err);
}
else {
res.json(200, response.hits.hits);
}
});
This search is retrieving all documents I have into the index.
Any help?
Thanks!
PD: Here is how I created index and mapping:
client.indices.create({ index: 'yojuego' }, (err, resp, respcode) => {
if (!err) {
client.indices.putMapping({
index: 'app',
type: "player",
body: {
properties: {
nickName: { type: "string" },
birthDate: { type: "string" },
state: { type: "string" },
adminState: { type: "string" },
account: {
type: "nested",
properties: {
id: { type: "string" },
type: { type: "string" },
password: { type: "string" }
}
}
}
}
}, (err, resp, respcode) => {
res.json(200, resp);
});
}
});
make sure that account is a nested field and then apply this query,
{
"query": {
"bool": {
"must": [
{
"nested": {
"path": "account",
"query": {
"bool": {
"must": [
{
"match": {
"account.id": "facundo#facundo"
}
},
{
"match": {
"account.type": "yojuego"
}
}
]
}
}
}
}
]
}
}
}
Related
I'm currently facing an issue with my datatable implemented in ReactJS. I'm retrieving data from elasticsearch and populating the datatable with it. The data retrieval process works fine without the filter applied, however, when I apply filters to the data, the datatable remains empty, even though the data _source has matching records.
The structure of the parameters I am sending is as follows:
{
pageIndex: 1,
pageSize: 10,
sort: { order: '', key: '' },
query: '',
filterData: {
analysis: [ '0', '1', '2', '3' ],
threat_level_id: [ '1', '2', '3', '4' ],
}
}
EndPoint:
POST /api/v1/events/public/list
Controller:
exports.getPublicEvents = async (req, res) => {
try {
client.ping()
const { pageIndex, pageSize, sort, query, filterData } = req.body
let esQuery = {
index: 'ns_*',
body: {
query: {
bool: {
must: [
{
match_all: {},
},
],
filter: [],
},
},
from: (pageIndex - 1) * pageSize,
size: pageSize,
},
}
if (query) {
esQuery.body.query.bool.must = [
{
match: {
'Event.info': {
query: query,
fuzziness: 'AUTO',
},
},
},
]
}
if (filterData.analysis.length > 0) {
esQuery.body.query.bool.filter.push({
terms: {
'Event.analysis': filterData.analysis,
},
})
}
if (filterData.threat_level_id.length > 0) {
esQuery.body.query.bool.filter.push({
terms: {
'Event.threat_level_id': filterData.threat_level_id,
},
})
}
let esResponse = await client.search(esQuery)
let data = esResponse.hits.hits.map((hit) => hit._source)
let total = esResponse.hits.total.value
res.status(200).json({
status: 'success',
data: data,
total: total,
})
} catch (error) {
res.status(500).json({
error: 'Error connecting to Elasticsearch',
errorMessage: error.message,
})
}
}
The controller below is without filters and it works just fine.
exports.getPublicEvents = async (req, res) => {
try {
client.ping()
const { pageIndex, pageSize, sort, query } = req.body
let esQuery = {
index: 'ns_*',
body: {
query: {
match_all: {},
},
from: (pageIndex - 1) * pageSize,
size: pageSize,
},
}
if (query) {
esQuery.body.query = {
match: {
'Event.info': {
query: query,
fuzziness: 'AUTO',
},
},
}
}
let esResponse = await client.search(esQuery)
let data = esResponse.hits.hits.map((hit) => hit._source)
let total = esResponse.hits.total.value
res.status(200).json({
status: 'success',
data: data,
total: total,
})
} catch (error) {
res.status(500).json({
error: 'Error connecting to Elasticsearch',
errorMessage: error.message,
})
}
}
ElasticSearech version: 7.17.8
Result of: console.log(JSON.stringify(esQuery))
{
"index": "INDEX_NAME",
"body": {
"query": {
"bool": {
"must": [{ "match_all": {} }],
"filter": [
{ "terms": { "Event.analysis": ["0", "1", "2"] } },
{ "terms": { "Event.threat_level_id": ["1", "2", "3", "4"] } }
]
}
},
"from": 0,
"size": 10
}
}
Data in elascticsearch schema
{
"#version": "1",
"#timestamp": "2023-02-01T14:43:09.997Z",
"Event": {
"info": ".......................",
"description": ".......................",
"analysis": 0,
"threat_level_id": "4",
"created_at": 1516566351,
"uuid": "5a64f74f0e543738c12bc973322",
"updated_at": 1675262417
}
}
Index Mapping
{
"index_patterns": ["INDEX_NAME"],
"template": "TEMPLATE_NAME",
"settings": {
"number_of_replicas": 0,
"index.mapping.nested_objects.limit": 10000000
},
"mappings": {
"dynamic": false,
"properties": {
"#timestamp": {
"type": "date"
},
"Event": {
"type": "nested",
"properties": {
"date_occured": {
"type": "date"
},
"threat_level_id": {
"type": "integer"
},
"description": {
"type": "text"
},
"is_shared": {
"type": "boolean"
},
"analysis": {
"type": "integer"
},
"uuid": {
"type": "text"
},
"created_at": {
"type": "date"
},
"info": {
"type": "text"
},
"shared_with": {
"type": "nested",
"properties": {
"_id": {
"type": "text"
}
}
},
"updated_at": {
"type": "date"
},
"author": {
"type": "text"
},
"Attributes": {
"type": "nested",
"properties": {
"data": {
"type": "text"
},
"type": {
"type": "text"
},
"uuid": {
"type": "text"
},
"comment": {
"type": "text"
},
"category": {
"type": "text"
},
"value": {
"type": "text"
},
"timestamp": {
"type": "date"
}
}
},
"organisation": {
"type": "nested",
"properties": {
"name": {
"type": "text"
},
"uuid": {
"type": "text"
}
}
},
"Tags": {
"type": "nested",
"properties": {
"color": {
"type": "text"
},
"name": {
"type": "text"
}
}
},
"TLP": {
"type": "nested",
"properties": {
"color": {
"type": "text"
},
"name": {
"type": "text"
}
}
}
}
}
}
}
}
Event is a nested field, so you need to use nested queries, like this:
{
"index": "INDEX_NAME",
"body": {
"query": {
"bool": {
"must": [{ "match_all": {} }],
"filter": [
{
"nested": {
"path": "Event",
"query": {"terms": { "Event.analysis": ["0", "1", "2"] }}
}
},
{
"nested": {
"path": "Event",
"query": {"terms": { "Event.threat_level_id": ["1", "2", "3", "4"] }}
}
}
]
}
},
"from": 0,
"size": 10
}
}
I need to remove the domain name from the URL in the JSON array stored in Mongo DB using Mongoose.
JSON Array format in DB
{
"_id": {
"$oid": "602a482223df2a16e26f51bc"
},
"message": "Test Again",
"pollType": {
"$numberInt": "1"
},
"type": "TEST",
"optionList": [
{
"name": "name",
"original": "https://name/test/pictures/1613383695668-71393-0d1f829222f945dc",
"thumbnail": "https://name/test/pictures/1613383695668-71393-0d1f829222f945dc",
"vote": {
"$numberInt": "1"
},
"_id": {
"$oid": "602a482223df2a16e26f51b9"
},
"participateUser": [
{
"$oid": "5c9baa00bcafa608891b0b44"
}
],
"latestParticipate": [
{
"$oid": "5c9baa00bcafa608891b0b44"
}
]
},
{
"name": null,
"original": "https://name/test/pictures/1613383695672-71393-e92a34eaf6b48b19",
"thumbnail": "https://name/test/pictures/1613383695672-71393-e92a34eaf6b48b19",
"vote": {
"$numberInt": "0"
},
"_id": {
"$oid": "602a482223df2a16e26f51ba"
},
"participateUser": [
]
},
{
"name": null,
"original": "https://name/test/pictures/1613383695630-71393-3387191491ba279c",
"thumbnail": "https://name/test/pictures/1613383695630-71393-3387191491ba279c",
"vote": {
"$numberInt": "0"
},
"_id": {
"$oid": "602a482223df2a16e26f51bb"
},
"participateUser": [
]
}
],
"pollId": {
"$oid": "602a482223df2a16e26f51b8"
},
"createdAt": {
"$date": {
"$numberLong": "1613383714396"
}
},
"updatedAt": {
"$date": {
"$numberLong": "1613383714396"
}
},
"totalVoteCount": {
"$numberInt": "1"
},
"participateUser": [
{
"$oid": "5c9baa00bcafa608891b0b44"
}
]
}
I am using below code to replace a string
return new Promise(async (resolve, reject) => {
console.log("Iam called")
MongoClient.connect('mongoDBconfig', function (err, client) {
if (err) throw err;
var collection = "test"
var db = client.db('testDB');
db.collection(collection).updateMany({
"optionList.thumbnail": { $regex: /name/ },
}, {
"$set": {
"optionList.$.thumbnail": {
$replaceOne: { input: "optionList.$.thumbnail", find: "https://name/test", replacement: "" },
}
}
}
}, function (error, success) {
if (success) {
resolve(success)
}
else {
reject(error)
}
})
})
})
but when I call this function I getting below error
DB Error : Duplicate Entry : dollar ($) prefixed field '$replaceOne' in 'optionList.0.original.$replaceOne' is not valid for storage.
The $replaceOne is a aggregation operator starting from MongoDB v4.4, you can use update with aggregation pipeline starting from MongoDB v4.2
$map to iterate loop of optionList array
$replaceOne to replace string
$mergeObjects to merge current object with updated original field
let originalName = "https://name/test";
db.collection(collection).updateMany(
{ "optionList.original": { $regex: originalName } },
[{
$set: {
optionList: {
$map: {
input: "$optionList",
in: {
$mergeObjects: [
"$$this",
{
thumbnail: {
$replaceOne: {
input: "$$this.thumbnail",
find: originalName,
replacement: ""
}
}
}
]
}
}
}
}
}],
function (error, success) {
if (success) { resolve(success) }
else { reject(error) }
}
)
Playground
I have this scheme
{
"_id": {
"$oid": "5e187b1791c51b4b105fcff0"
},
"username": "test",
"email": "test#test.com",
"role": "admin",
"userScoreFantasy": {
"tournaments": [
{
"tournament_id": {
"$oid": "5e1892fb480f344830a3f160"
},
"predictions": [],
"score": null
},
{
"tournament_id": {
"$oid": "5e189f5f8d88292754e10b37"
},
"predictions": [],
"score": null
}
],
"totalScore": null
},
}
I want to do this :
Find user with a predefined user id
Pass all userScoreFantasy.tournaments array to find a specific tournament id
Push into the found tournament predictions array an object like this one :
{
score,
"match_id": foundMatch._id
}
So the tournaments array will become :
[
{
"tournament_id": {
"$oid": "5e1892fb480f344830a3f160"
},
"predictions": [
"score" : 200,
"match_id": "5e1892fb480f34faw21410"
],
"score": null
},
]
I tried to do this :
Users.update({
"_id": prediction.user_id,
"userScoreFantasy.tournaments": {
"$elemMatch": {
"tournament_id": foundMatch.tournament_id
}
}
}, {
"$push": {
"userScoreFantasy.tournaments.$.predictions": {
score,
"match_id": foundMatch._id
}
}
})
But the array is not updating.
EDIT :
Working call :
Users.updateOne({
"_id": ObjectID(prediction.user_id),
}, {
$addToSet: {
"userScoreFantasy.tournaments.$[element].predictions": {
score,
"match_id": foundMatch._id
}
}
}, {
arrayFilters: [{
"element.tournament_id": ObjectID(foundMatch.tournament_id)
}]
}
)
You should use the position indentifier to update your arrays, like so:
Users.updateOne(
{
"_id": prediction.user_id,
},
{
$addToSet: {
"userScoreFantasy.tournaments.$[element].predictions": {
score,
"match_id": foundMatch._id
}
}
},
{arrayFilters: [{"element.tournament_id": foundMatch.tournament_id}]}
);
How to get suggested query values from elasticsearch nodejs client.
I have such kind of index named test :
PUT test
{
"settings": {
"index": {
"number_of_shards": 1,
"analysis": {
"analyzer": {
"trigram": {
"type": "custom",
"tokenizer": "standard",
"filter": ["lowercase","shingle"]
},
"reverse": {
"type": "custom",
"tokenizer": "standard",
"filter": ["lowercase","reverse"]
}
},
"filter": {
"shingle": {
"type": "shingle",
"min_shingle_size": 2,
"max_shingle_size": 3
}
}
}
}
},
"mappings": {
"properties": {
"word": {
"type": "text",
"fields": {
"trigram": {
"type": "text",
"analyzer": "trigram"
},
"reverse": {
"type": "text",
"analyzer": "reverse"
}
}
}
}
}
}
and want to take suggest.simple_phrase values by the help of elasticsearch client by this way:
getElasticSearchResult: (_, params, context) =>
new Promise((resolve, reject) => {
const param: RequestParams.Search = {
index: 'test',
body: {
suggest: {
text: 'params.searchText',
simple_phrase: {
phrase: {
field: 'word.trigram',
size: 1,
gram_size: 3,
direct_generator: [
{
field: 'word.trigram',
suggest_mode: 'always',
},
],
highlight: {
pre_tag: '<em>',
post_tag: '</em>',
},
},
},
},
},
};
client
.search(param)
.then((result: ApiResponse) => {
resolve(result.body.suggest.simple_phrase);
console.log('---- ', resolve(result.body));
// keep results in array (allRecords)
})
.catch((err: Error) => {
console.log(err);
reject(err);
});
}).then(v => {
return allRecords;
}),
},
but result.body is undefined and why?
request comes correctly and result structure is also correct but inside everything is empty.
I'm trying to retrieve some data from ElasticSearch.
So far everything is working perfectly and I can query data.
But whenever I try to count a field using aggregations, the aggregation field is not in the result at the end.
So far what I've tried this as my query/function :
var client = new elasticsearch.Client({
host: 'xxxxxxxxxxxxxxxxxxxxxxx',
log:"trace"
});
client.ping({
requestTimeout: 30000,
}, function (error) {
if (error) {
console.error('elasticsearch cluster is down!');
} else {
console.log('All is well');
}
});
client.search({
"index":"worklight__appsession__1485302400000",
"type":"AppSession",
"body":{
"query": {
"filtered": {
"query": {
"query_string": {
"analyze_wildcard": true,
"query": "*"
}
},
"filter": {
"bool": {
"must": [
{
"range": {
"timestamp": {
"gte": 1553507131976,
"lte": 1553593531976
}
}
}
],
"must_not": []
}
}
}
},
"aggs": {
"1": {
"cardinality": {
"field": "deviceID"
}
}
}
}
}).then(function (resp) {
var hits = resp.hits.hits;
console.log(hits)
}, function (err) {
console.trace(err.message);
});
and the result is :
Elasticsearch DEBUG: 2019-03-26T09:46:21Z
starting request {
"method": "HEAD",
"requestTimeout": 30000,
"castExists": true,
"path": "/",
"query": {}
}
Elasticsearch DEBUG: 2019-03-26T09:46:21Z
starting request {
"method": "POST",
"path": "/worklight__appsession__1485302400000/AppSession/_search",
"body": {
"query": {
"filtered": {
"query": {
"query_string": {
"analyze_wildcard": true,
"query": "*"
}
},
"filter": {
"bool": {
"must": [
{
"range": {
"timestamp": {
"gte": 1553507131976,
"lte": 1553593531976
}
}
}
],
"must_not": []
}
}
}
},
"aggs": {
"1": {
"cardinality": {
"field": "deviceID"
}
}
}
},
"query": {}
}
Elasticsearch TRACE: 2019-03-26T09:46:22Z
-> HEAD http://xx/
<- 200
Elasticsearch DEBUG: 2019-03-26T09:46:22Z
Request complete
All is well
Elasticsearch TRACE: 2019-03-26T09:46:22Z
-> POST http://xx/worklight__appsession__1485302400000/AppSession/_search
{
"query": {
"filtered": {
"query": {
"query_string": {
"analyze_wildcard": true,
"query": "*"
}
},
"filter": {
"bool": {
"must": [
{
"range": {
"timestamp": {
"gte": 1553507131976,
"lte": 1553593531976
}
}
}
],
"must_not": []
}
}
}
},
"aggs": {
"1": {
"cardinality": {
"field": "deviceID"
}
}
}
}
<- 200
{
"took": 4,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 325,
"max_score": 1,
"hits": [
...
confidential data here, not relevant to the topic.
...
}
]
},
"aggregations": {
"1": {
"value": 133
}
}
}
But if erase the log trace option, aggregations don't show up in the result :
[ { _index: 'worklight__appsession__1485302400000',
_type: 'AppSession',
... Some Data,
{ _index: 'worklight__appsession__1485302400000',
_type: 'AppSession',
... Some Data,
{ _index: 'worklight__appsession__1485302400000',
_type: 'AppSession',
... Some Data,
]
Am I doing something wrong, or do I just lack knowledge ?
Thanks for your time.
You are doing console.log(resp.hits.hits). Try this instead:
console.log(resp.aggregations)