ElasticSearch query works on dev console but fails in NodeJS instance - node.js

I have the following ElasticSearch query that used to retrieve a bunch of pages (search) paginated using a cursor offset.
Query
GET _search
{
"search_after": [
1.8574909,
"urn:sample/78PsC1EHG6nopQCA/n/749d1ed1-d08d-44a1-abac-9ebad8c76697"
],
"sort": [
{
"_score": {
"order": "desc"
}
},
{
"_id": {
"order": "asc"
}
}
],
"size": 1,
"query": {
"bool": {
"must": [
{
"bool": {
"should": [
{
"bool": {
"must": [
{
"term": {
"type": "node"
}
},
{
"query_string": {
"query": "a",
"fields": [
"node.text.string",
"node.text.string.english"
]
}
}
]
}
}
]
}
}
],
"filter": {
"bool": {
"should": [
{
"bool": {
"must": [
{
"term": {
"workspace.type": "space"
}
},
{
"terms": {
"workspace.id": [
"72MsGpeV9zGu5ytZ"
]
}
}
]
}
},
{
"bool": {
"must": [
{
"term": {
"workspace.type": "user"
}
},
{
"term": {
"workspace.id": "8"
}
}
]
}
}
]
}
}
}
},
"highlight": {
"fields": {
"comment.body.text": {},
"comment.body.text.english": {},
"node.text.string": {},
"node.text.string.english": {}
}
}
}
The query works fine in the dev console and returns a hit with an edge.
Result
{
"took" : 136,
"timed_out" : false,
"num_reduce_phases" : 2,
"_shards" : {
"total" : 929,
"successful" : 929,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : 5,
"max_score" : null,
"hits" : [
{
"_index" : "sample-development-john",
"_type" : "_doc",
"_id" : "urn:sample:d/78PsC1EHG6nopQCA/n/4230ca5a-5e1e-48ee-a70f-95f7b2d09995",
"_score" : 1.7747711,
"_source" : {
"document_id" : "78PsC1EHG6nopQCA",
"type" : "node",
"workspace" : {
"type" : "space",
...
However, performing the same query through my NodeJS resolver using the Javascript ES API fails when I copy the exact same query in my code.
SearchResolver.js
// parse the cursor argument for pagination.
search_after = JSON.parse(Buffer.from(args.after, 'base64').toString('ascii'));
const response: ApiResponse<Taskade.Backend.Elasticsearch.SearchBody> = await client.search({
index,
body: {
search_after: [
1.8574909,
'urn:taskade:d/78PsC1EHG6nopQCA/n/749d1ed1-d08d-44a1-abac-9ebad8c76697',
],
sort: [
{
_score: {
order: 'desc',
},
},
{
_id: {
order: 'asc',
},
},
],
size: 1,
query: {
bool: {
must: [
{
bool: {
should: [
{
bool: {
must: [
{
term: {
type: 'node',
},
},
{
query_string: {
query: 'a',
fields: ['node.text.string', 'node.text.string.english'],
},
},
],
},
},
],
},
},
],
filter: {
bool: {
should: [
{
bool: {
must: [
{
term: {
'workspace.type': 'space',
},
},
{
terms: {
'workspace.id': ['72MsGpeV9zGu5ytZ'],
},
},
],
},
},
{
bool: {
must: [
{
term: {
'workspace.type': 'user',
},
},
{
term: {
'workspace.id': '8',
},
},
],
},
},
],
},
},
},
},
highlight: {
fields: {
'comment.body.text': {},
'comment.body.text.english': {},
'node.text.string': {},
'node.text.string.english': {},
},
},
},
}
Error
{
"errors": [
{
"message": "parsing_exception",
"locations": [
{
"line": 2,
"column": 3
}
],
"path": [
"search"
],
"extensions": {
"code": "INTERNAL_SERVER_ERROR",
"exception": {
"name": "ResponseError",
"meta": {
"body": {
"error": {
"root_cause": [
{
"type": "parsing_exception",
"reason": "Unknown key for a VALUE_STRING in [query].",
"line": 1,
"col": 10
}
],
"type": "parsing_exception",
"reason": "Unknown key for a VALUE_STRING in [query].",
"line": 1,
"col": 10
},
"status": 400
},
"statusCode": 400,
"headers": {
"date": "Mon, 28 Dec 2020 11:58:05 GMT",
"content-type": "application/json; charset=UTF-8",
"content-length": "239",
"connection": "keep-alive",
"access-control-allow-origin": "*"
},
"warnings": null,
"meta": {
"context": null,
"request": {
"params": {
"method": "POST",
"path": "/sample-development-john/_search",
"body": "{\"query\":\"a\",\"size\":1,\"sort\":[{\"_score\":{\"order\":\"asc\"}},{\"_id\":{\"order\":\"desc\"}}],\"search_after\":[1.7747711,\"urn:sample:d/78PsC1EHG6nopQCA/n/4230ca5a-5e1e-48ee-a70f-95f7b2d09995\"]}",
"querystring": "",
"headers": {
"User-Agent": "elasticsearch-js/6.8.7 (darwin 20.2.0-x64; Node.js v10.21.0)",
"Content-Type": "application/json",
"Content-Length": "182"
},
"timeout": 30000
},
"options": {
"warnings": null
},
"id": 2
},
"name": "elasticsearch-js",
"connection": {
"url": "https://search-sample-dev-526nv5wyqxj6ahzcql2cyndz5e.us-east-1.es.amazonaws.com/",
"id": "https://search-sample-dev-526nv5wyqxj6ahzcql2cyndz5e.us-east-1.es.amazonaws.com/",
"headers": {},
"deadCount": 0,
"resurrectTimeout": 0,
"_openRequests": 0,
"status": "alive",
"roles": {
"master": true,
"data": true,
"ingest": true,
"ml": false
}
},
"attempts": 0,
"aborted": false
}
},
"stacktrace": [
"ResponseError: parsing_exception",
" at IncomingMessage.response.on (/Users/john/dev/sample/ft/node_modules/#elastic/elasticsearch/lib/Transport.js:296:25)",
" at IncomingMessage.emit (events.js:203:15)",
" at IncomingMessage.EventEmitter.emit (domain.js:466:23)",
" at endReadableNT (_stream_readable.js:1145:12)",
" at process._tickCallback (internal/process/next_tick.js:63:19)"
]
}
}
}
],
"data": {
"search": null
}
}
This error occurs when I pass the after (search_after) parameter to my GraphQL call, despite the values for the query all being hardcoded (including search_after as seen in the first Query snippet).
// query will work if `after` is not passed. The values here don't matter since for testing purposes, all values are hardcoded in the resolver.
query {
search(after: "WzEuODU3NDkwOSwidXJuOnRhc2thZGU6ZC83OFBzQzFFSEc2bm9wUUNBL24vNzQ5ZDFlZDEtZDA4ZC00NGExLWFiYWMtOWViYWQ4Yzc2Njk3Il0=", first:1, filterby:{query:"a"}){
edges{
cursor
node {
...
Why is the query working in the dev console, but not working in my resolver when I pass the after (search_after after parsing the JSON) params to the GQL call despite the query being hardcoded and identically similar.

Related

Filter data using nodejs and elasticsearch

I'm currently facing an issue with my datatable implemented in ReactJS. I'm retrieving data from elasticsearch and populating the datatable with it. The data retrieval process works fine without the filter applied, however, when I apply filters to the data, the datatable remains empty, even though the data _source has matching records.
The structure of the parameters I am sending is as follows:
{
pageIndex: 1,
pageSize: 10,
sort: { order: '', key: '' },
query: '',
filterData: {
analysis: [ '0', '1', '2', '3' ],
threat_level_id: [ '1', '2', '3', '4' ],
}
}
EndPoint:
POST /api/v1/events/public/list
Controller:
exports.getPublicEvents = async (req, res) => {
try {
client.ping()
const { pageIndex, pageSize, sort, query, filterData } = req.body
let esQuery = {
index: 'ns_*',
body: {
query: {
bool: {
must: [
{
match_all: {},
},
],
filter: [],
},
},
from: (pageIndex - 1) * pageSize,
size: pageSize,
},
}
if (query) {
esQuery.body.query.bool.must = [
{
match: {
'Event.info': {
query: query,
fuzziness: 'AUTO',
},
},
},
]
}
if (filterData.analysis.length > 0) {
esQuery.body.query.bool.filter.push({
terms: {
'Event.analysis': filterData.analysis,
},
})
}
if (filterData.threat_level_id.length > 0) {
esQuery.body.query.bool.filter.push({
terms: {
'Event.threat_level_id': filterData.threat_level_id,
},
})
}
let esResponse = await client.search(esQuery)
let data = esResponse.hits.hits.map((hit) => hit._source)
let total = esResponse.hits.total.value
res.status(200).json({
status: 'success',
data: data,
total: total,
})
} catch (error) {
res.status(500).json({
error: 'Error connecting to Elasticsearch',
errorMessage: error.message,
})
}
}
The controller below is without filters and it works just fine.
exports.getPublicEvents = async (req, res) => {
try {
client.ping()
const { pageIndex, pageSize, sort, query } = req.body
let esQuery = {
index: 'ns_*',
body: {
query: {
match_all: {},
},
from: (pageIndex - 1) * pageSize,
size: pageSize,
},
}
if (query) {
esQuery.body.query = {
match: {
'Event.info': {
query: query,
fuzziness: 'AUTO',
},
},
}
}
let esResponse = await client.search(esQuery)
let data = esResponse.hits.hits.map((hit) => hit._source)
let total = esResponse.hits.total.value
res.status(200).json({
status: 'success',
data: data,
total: total,
})
} catch (error) {
res.status(500).json({
error: 'Error connecting to Elasticsearch',
errorMessage: error.message,
})
}
}
ElasticSearech version: 7.17.8
Result of: console.log(JSON.stringify(esQuery))
{
"index": "INDEX_NAME",
"body": {
"query": {
"bool": {
"must": [{ "match_all": {} }],
"filter": [
{ "terms": { "Event.analysis": ["0", "1", "2"] } },
{ "terms": { "Event.threat_level_id": ["1", "2", "3", "4"] } }
]
}
},
"from": 0,
"size": 10
}
}
Data in elascticsearch schema
{
"#version": "1",
"#timestamp": "2023-02-01T14:43:09.997Z",
"Event": {
"info": ".......................",
"description": ".......................",
"analysis": 0,
"threat_level_id": "4",
"created_at": 1516566351,
"uuid": "5a64f74f0e543738c12bc973322",
"updated_at": 1675262417
}
}
Index Mapping
{
"index_patterns": ["INDEX_NAME"],
"template": "TEMPLATE_NAME",
"settings": {
"number_of_replicas": 0,
"index.mapping.nested_objects.limit": 10000000
},
"mappings": {
"dynamic": false,
"properties": {
"#timestamp": {
"type": "date"
},
"Event": {
"type": "nested",
"properties": {
"date_occured": {
"type": "date"
},
"threat_level_id": {
"type": "integer"
},
"description": {
"type": "text"
},
"is_shared": {
"type": "boolean"
},
"analysis": {
"type": "integer"
},
"uuid": {
"type": "text"
},
"created_at": {
"type": "date"
},
"info": {
"type": "text"
},
"shared_with": {
"type": "nested",
"properties": {
"_id": {
"type": "text"
}
}
},
"updated_at": {
"type": "date"
},
"author": {
"type": "text"
},
"Attributes": {
"type": "nested",
"properties": {
"data": {
"type": "text"
},
"type": {
"type": "text"
},
"uuid": {
"type": "text"
},
"comment": {
"type": "text"
},
"category": {
"type": "text"
},
"value": {
"type": "text"
},
"timestamp": {
"type": "date"
}
}
},
"organisation": {
"type": "nested",
"properties": {
"name": {
"type": "text"
},
"uuid": {
"type": "text"
}
}
},
"Tags": {
"type": "nested",
"properties": {
"color": {
"type": "text"
},
"name": {
"type": "text"
}
}
},
"TLP": {
"type": "nested",
"properties": {
"color": {
"type": "text"
},
"name": {
"type": "text"
}
}
}
}
}
}
}
}
Event is a nested field, so you need to use nested queries, like this:
{
"index": "INDEX_NAME",
"body": {
"query": {
"bool": {
"must": [{ "match_all": {} }],
"filter": [
{
"nested": {
"path": "Event",
"query": {"terms": { "Event.analysis": ["0", "1", "2"] }}
}
},
{
"nested": {
"path": "Event",
"query": {"terms": { "Event.threat_level_id": ["1", "2", "3", "4"] }}
}
}
]
}
},
"from": 0,
"size": 10
}
}

Missing date data when getting data from Google Analytic 4 (GA4) to create chart with granularity daily?

I'm getting started with Google Analytics 4, and I have a task is creating chart report with custom dimensions. I have a problem when get data from API. It missing date data when getting data from GA4. I want to use this data to build a chart
const {
propertyId,
startDate,
endDate,
} = req.query;
// Request report data.
const analyticsDataClient = await getAnalyticsDataClient(getActiveShop(req));
const response = await analyticsDataClient.runPivotReport({
property: `properties/${propertyId}`,
dimensions: [
{ name: "date" },
{ name: "CUSTOM_DIMENSION_EVENT" },
{ name: "CUSTOM_DIMENSION_EVENT" },
{ name: "CUSTOM_DIMENSION_EVENT" },
],
metrics: [
{ name: "sessions" },
{ name: "activeUsers" },
{ name: "CUSTOM_METRIC_EVENT" },
{ name: "CUSTOM_METRIC_EVENT" },
],
dateRanges: [{ startDate: 2022-12-06, endDate: 2023-01-03 }], // Dateranges
pivots: [
{
fieldNames: ["date"],
limit: 30,
},
{
fieldNames: ["CUSTOM_DIMENSION_EVENT"],
limit: 5,
},
{
fieldNames: ["CUSTOM_DIMENSION_EVENT"],
limit: 5,
},
{
fieldNames: ["CUSTOM_DIMENSION_EVENT"],
limit: 5,
},
],
dimensionFilter: {
filter: {
fieldName: "CUSTOM_METRIC_EVENT", // id
stringFilter: {
matchType: "EXACT",
value: "CUSTOM_METRIC_EVENT_ID",
},
},
},
keepEmptyRows: true,
});
response GA4 API
{
"response": [
{
"pivotHeaders": [
{
"pivotDimensionHeaders": [
{
"dimensionValues": [
{
"value": "20221227", // return this date only
"oneValue": "value"
}
]
}
],
"rowCount": 1
},
{
"pivotDimensionHeaders": [
{
"dimensionValues": [
{
"value": "885c7b0d-bc65-47be-b7df-871947bc5de4",
"oneValue": "value"
}
]
}
],
"rowCount": 1
},
{
"pivotDimensionHeaders": [
{
"dimensionValues": [
{
"value": "New Page For Test GA4",
"oneValue": "value"
}
]
}
],
"rowCount": 1
},
{
"pivotDimensionHeaders": [
{
"dimensionValues": [
{
"value": "index",
"oneValue": "value"
}
]
}
],
"rowCount": 1
}
],
"dimensionHeaders": [
{
"name": "date"
},
{
"name": "customEvent:page_id"
},
{
"name": "customEvent:page_name"
},
{
"name": "customEvent:page_type"
}
],
"metricHeaders": [
{
"name": "sessions",
"type": "TYPE_INTEGER"
},
{
"name": "activeUsers",
"type": "TYPE_INTEGER"
},
{
"name": "customEvent:revenue",
"type": "TYPE_CURRENCY"
}
],
"rows": [
{
"dimensionValues": [
{
"value": "20221227",
"oneValue": "value"
},
{
"value": "885c7b0d-bc65-47be-b7df-871947bc5de4",
"oneValue": "value"
},
{
"value": "New Page For Test GA4",
"oneValue": "value"
},
{
"value": "index",
"oneValue": "value"
}
],
"metricValues": [
{
"value": "7",
"oneValue": "value"
},
{
"value": "7",
"oneValue": "value"
},
{
"value": "0",
"oneValue": "value"
},
{
"value": "8",
"oneValue": "value"
}
]
}
],
"aggregates": [],
"metadata": {
"dataLossFromOtherRow": false,
"schemaRestrictionResponse": {
"activeMetricRestrictions": []
},
"_schemaRestrictionResponse": "schemaRestrictionResponse",
"_currencyCode": "currencyCode",
"timeZone": "America/Los_Angeles",
"_timeZone": "timeZone"
},
"propertyQuota": null,
"kind": "analyticsData#runPivotReport"
},
null,
null
]
}
Although I set dateRanges { startDate: 2022-12-06, endDate: 2023-01-03 } , it only returns 20221227 for me? What's problem here?
If you check runPivotReport in the doucmentation you will find that the response body for a RunPivotReportResponse
Does not contain dates, the api probably assumes you know what dates you requested.
{
"pivotHeaders": [
{
object (PivotHeader)
}
],
"dimensionHeaders": [
{
object (DimensionHeader)
}
],
"metricHeaders": [
{
object (MetricHeader)
}
],
"rows": [
{
object (Row)
}
],
"aggregates": [
{
object (Row)
}
],
"metadata": {
object (ResponseMetaData)
},
"propertyQuota": {
object (PropertyQuota)
},
"kind": string
}

Opensearch transform splitting array values to new events

I'm transforming index that contains following event.
But the values inside of array are splitting into the new events.
e.g:
"serviceIdentifiers": "Redis"
"serviceIdentifiers":"Event_Detector Servicc"
etc.
{
"_index": "collated_txn_health_2022.05",
"_type": "_doc",
"_id": "LAUpboIBh6CUatILrsN3",
"_score": 1,
"_source": {
"timeInGMT": 0,
"kpiId": 0,
"compInstanceIdentifier": "d0352b7d-0484-4714-bbc8-eb67cbb7be70",
"agentIdentifier": "ComponentAgent-171",
"kpiIdentifier": "PACKETS_DROPPED",
"categoryIdentifier": "Network Utilization",
"applicationIdentifier": null,
"serviceIdentifiers": [
"Supervisor_Controller Service",
"Event_Detector Service",
"UI_Service",
"Redis",
"CC_Service"
],
"clusterIdentifiers": [
"a5c57ef5-4018-41b8-b727-27c8f8376c0e"
],
"collectionInterval": 60,
"value": "0.0",
"kpiType": "Core",
"groupAttribute": "ALL",
"groupIdentifier": null,
"watcherValue": null,
"errorCode": null,
"clusterOperation": null,
"aggLevelInMins": 1,
"error": false,
"kpiGroup": false,
"discovery": false,
"maintenanceExcluded": false,
"#timestamp": "2022-05-01T01:32:00.000Z"
}
Following is the transform job configuration.
curl -u admin:admin -XPUT "http://XXX.XXX.XX.XXX9201/_plugins/_transform/my-array-job-2" -H 'Content-type: application/json' -d'
{
"transform": {
"schedule": {
"interval": {
"start_time": 1659705000000,
"period": 1,
"unit": "Minutes"
}
},
"metadata_id": null,
"updated_at": 1659456180000,
"enabled": true,
"enabled_at": 1659457620000,
"description": "",
"source_index": "collated_txn_health_2022.05",
"data_selection_query": {
"match_all": {
"boost": 1
}
},
"target_index": "transform_collated_txn_health_2022.05",
"page_size": 1000,
"groups": [
{
"date_histogram": {
"fixed_interval": "1m",
"source_field": "#timestamp",
"target_field": "#timestamp",
"timezone": "Asia/Calcutta"
}
},
{
"terms": {
"source_field": "clusterIdentifiers",
"target_field": "clusterIdentifiers"
}
},
{
"terms": {
"source_field": "serviceIdentifiers",
"target_field": "serviceIdentifiers"
}
},
{
"terms": {
"source_field": "compInstanceIdentifier",
"target_field": "compInstanceIdentifier"
}
},
{
"terms": {
"source_field": "agentIdentifier",
"target_field": "agentIdentifier"
}
}
],
"aggregations": {
"count_#timestamp": {
"value_count": {
"field": "#timestamp"
}
}
}
}
}'
Following are the events from the transform index.
{
"_index": "transform_heal_collated_txn_health_2022.05",
"_type": "_doc",
"_id": "ybK0McQ9NZrt9xdo9iWKbA",
"_score": 1,
"_source": {
"transform._id": "my-array-job-2",
"transform._doc_count": 2,
"#timestamp": 1651365120000,
"clusterIdentifiers": "a5c57ef5-4018-41b8-b727-27c8f8376c0e",
"serviceIdentifiers": "Redis",
"compInstanceIdentifier": "a5c57ef5-4018-41b8-b727-27c8f8376c0e",
"agentIdentifier": "ComponentAgent-170",
"count_#timestamp": 2
}
},
{
"_index": "transform_heal_collated_txn_health_2022.05",
"_type": "_doc",
"_id": "Wf-4KwnFaYuw9bL-V-9WEQ",
"_score": 1,
"_source": {
"transform._id": "my-array-job-2",
"transform._doc_count": 2,
"#timestamp": 1651365120000,
"clusterIdentifiers": "a5c57ef5-4018-41b8-b727-27c8f8376c0e",
"serviceIdentifiers": "Redis_Server Service",
"compInstanceIdentifier": "a5c57ef5-4018-41b8-b727-27c8f8376c0e",
"agentIdentifier": "ComponentAgent-170",
"count_#timestamp": 2
}
It would be a great help if somebody suggest me with solution for array fields.
Have solved the issue with following painless script. Which help to transform array fields in opensearch.
PUT _plugins/_transform/my-array-job-2
{
"transform": {
"schedule": {
"interval": {
"start_time": 1659705000000,
"period": 1,
"unit": "Minutes"
}
},
"metadata_id": null,
"updated_at": 1659456180000,
"enabled": true,
"enabled_at": 1659457620000,
"description": "",
"source_index": "heal_collated_txn_heal_health_2022.05_reindex",
"target_index": "transform_heal_collated_txn_heal_health_2022.05",
"page_size": 1000,
"groups": [
{
"date_histogram": {
"fixed_interval": "1m",
"source_field": "#timestamp",
"target_field": "#timestamp",
"timezone": "Asia/Calcutta"
}
},
{
"terms": {
"source_field": "kpiIdentifier",
"target_field": "kpiIdentifier"
}
},
{
"terms": {
"source_field": "clusterIdentifiers",
"target_field": "clusterIdentifiers"
}
}
],
"aggregations": {
"count_#timestamp": {
"value_count": {
"field": "#timestamp"
}
},
"count_agentIdentifier": {
"value_count": {
"field": "agentIdentifier"
}
},
"sum_value": {
"sum": {
"field": "value"
}
},
"max_value": {
"max": {
"field": "value"
}
},
"avg_value": {
"avg": {
"field": "value"
}
},
"count_value": {
"value_count": {
"field": "value"
}
},
"percentiles_value": {
"percentiles": {
"field": "value",
"percents": [
95
],
"keyed": true,
"tdigest": {
"compression": 100
}
}
},
"serviceIdentifiers": {
"scripted_metric": {
"init_script": "state.docs = []",
"map_script": """
Map span = [
'url':doc['serviceIdentifiers']
];
state.docs.add(span)
""",
"combine_script": "return state.docs;",
"reduce_script": """
def all_docs = [];
for (s in states) {
for (span in s) {
all_docs.add(span);
}
}
def size = all_docs.size();
def serviceIdentifiers_1 = all_docs[0]['url'];
def ret = new HashMap();
ret['serviceIdentifiers'] = serviceIdentifiers_1;
return ret;
"""
}
}
}
}
}

Filter nested result inside a nested object with elasticsearch

I'm trying to filter a nested object and sort by the result, however, I tried some things without success, I'll leave my initial attempt and it works partially, it just filters according to what I have in my search variable, but all the results come of this nested object as it is inside the 'root' which is another nested object
Elastic version: 7.13.0 with NodeJS
using #elastic/elasticsearch official package from npm
let params: RequestParams.Search = {
index: index,
body: {
size: 30,
query: {
bool: {
must: [
{
nested: {
path: "profile",
query: {
bool: {
must: [
{
match: {
"profile.id": profileId,
},
},
],
},
},
},
},
],
filter: [
{
nested: {
path: "profile.following",
ignore_unmapped: true,
query: {
query_string: {
fields: [
"profile.following.name",
"profile.following.username",
],
query: searchWord + "*",
},
},
},
},
],
},
},
},
};
I need it to be this specific 'profile.id' that is passed by parameter in the function, so the result is only 1 profile with N people that it follows
the document is mapped as follows, I left only the fields relevant to the question:
{
"mappings": {
"_doc": {
"properties": {
"id": {
"type": "integer"
},
"phone": {
"type": "text"
},
"profile": {
"type": "nested",
"properties": {
"id": {
"type": "integer"
},
"username": {
"type": "text"
},
"following": {
"type": "nested",
"properties": {
"id": {
"type": "integer"
},
"isAwaitingApproval": {
"type": "boolean"
},
"name": {
"type": "text"
},
"profilePicURL": {
"type": "text"
},
"username": {
"type": "text"
}
}
}
}
}
}
}
}
}
an example of a current result is:
with the following parameters (profileId:141, searchWord: "para" )
{
"res": [
{
"profilePicURL": "localimage",
"name": "donor donor",
"id": 140,
"username": "victorTesteElastic2",
"isAwaitingApproval": false
},
{
"profilePicURL": "localimage",
"name": "para ser seguido",
"id": 142,
"username": "victorprivate",
"isAwaitingApproval": true
}
]
}
the desired result is:
{
"res": [
{
"profilePicURL": "localimage",
"name": "para ser seguido",
"id": 142,
"username": "victorprivate",
"isAwaitingApproval": true
}
]
}
with some more research I got what I needed, I'll leave the answer here in case anyone needs it too
let params: RequestParams.Search = {
index: index,
body: {
size: 30,
query: {
bool: {
must: [
{
nested: {
path: "profile",
query: {
bool: {
must: [
{
match: {
"profile.id": profileId,
},
},
],
},
},
},
},
{
nested: {
path: "profile",
inner_hits: {
name: "profile",
},
query: {
nested: {
path: "profile.following",
inner_hits: {
name: "following",
},
ignore_unmapped: true,
query: {
query_string: {
fields: [
"profile.following.name",
"profile.following.username",
],
query: searchWord + "*",
},
},
},
},
},
},
],
},
},
},
};
I basically put in must what was in the filter, mapped the nested object from above, in this case the profile, and put the tag inner_hits for profile and inner_hits for followings, that's the only way it worked
the answer I need was returned here:
body.hits.hits[0].inner_hits.profile.hits.hits[0].inner_hits.following.hits.hits
below is an example of the answer:
{
"res": [
{
"_index": "donor",
"_type": "_doc",
"_id": "P3VWNnsB4coAEhD-F3fF",
"_nested": {
"field": "profile",
"offset": 0,
"_nested": {
"field": "following",
"offset": 0
}
},
"_score": 1,
"_source": {
"profilePicURL": "localimage",
"name": "donor donor",
"id": 140,
"username": "victorTesteElastic2",
"isAwaitingApproval": false
}
},
{
"_index": "donor",
"_type": "_doc",
"_id": "P3VWNnsB4coAEhD-F3fF",
"_nested": {
"field": "profile",
"offset": 0,
"_nested": {
"field": "following",
"offset": 1
}
},
"_score": 1,
"_source": {
"profilePicURL": "localimage",
"name": "para ser seguido",
"id": 142,
"username": "victorprivate",
"isAwaitingApproval": true
}
}
]
}
the filtered data I really need that have been matched in must is in this array, where I need to iterate and look at _source which is the data that is indexed

Is there a way with ElasticSearch to only show aggregations?

I'm trying to retrieve some data from ElasticSearch.
So far everything is working perfectly and I can query data.
But whenever I try to count a field using aggregations, the aggregation field is not in the result at the end.
So far what I've tried this as my query/function :
var client = new elasticsearch.Client({
host: 'xxxxxxxxxxxxxxxxxxxxxxx',
log:"trace"
});
client.ping({
requestTimeout: 30000,
}, function (error) {
if (error) {
console.error('elasticsearch cluster is down!');
} else {
console.log('All is well');
}
});
client.search({
"index":"worklight__appsession__1485302400000",
"type":"AppSession",
"body":{
"query": {
"filtered": {
"query": {
"query_string": {
"analyze_wildcard": true,
"query": "*"
}
},
"filter": {
"bool": {
"must": [
{
"range": {
"timestamp": {
"gte": 1553507131976,
"lte": 1553593531976
}
}
}
],
"must_not": []
}
}
}
},
"aggs": {
"1": {
"cardinality": {
"field": "deviceID"
}
}
}
}
}).then(function (resp) {
var hits = resp.hits.hits;
console.log(hits)
}, function (err) {
console.trace(err.message);
});
and the result is :
Elasticsearch DEBUG: 2019-03-26T09:46:21Z
starting request {
"method": "HEAD",
"requestTimeout": 30000,
"castExists": true,
"path": "/",
"query": {}
}
Elasticsearch DEBUG: 2019-03-26T09:46:21Z
starting request {
"method": "POST",
"path": "/worklight__appsession__1485302400000/AppSession/_search",
"body": {
"query": {
"filtered": {
"query": {
"query_string": {
"analyze_wildcard": true,
"query": "*"
}
},
"filter": {
"bool": {
"must": [
{
"range": {
"timestamp": {
"gte": 1553507131976,
"lte": 1553593531976
}
}
}
],
"must_not": []
}
}
}
},
"aggs": {
"1": {
"cardinality": {
"field": "deviceID"
}
}
}
},
"query": {}
}
Elasticsearch TRACE: 2019-03-26T09:46:22Z
-> HEAD http://xx/
<- 200
Elasticsearch DEBUG: 2019-03-26T09:46:22Z
Request complete
All is well
Elasticsearch TRACE: 2019-03-26T09:46:22Z
-> POST http://xx/worklight__appsession__1485302400000/AppSession/_search
{
"query": {
"filtered": {
"query": {
"query_string": {
"analyze_wildcard": true,
"query": "*"
}
},
"filter": {
"bool": {
"must": [
{
"range": {
"timestamp": {
"gte": 1553507131976,
"lte": 1553593531976
}
}
}
],
"must_not": []
}
}
}
},
"aggs": {
"1": {
"cardinality": {
"field": "deviceID"
}
}
}
}
<- 200
{
"took": 4,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 325,
"max_score": 1,
"hits": [
...
confidential data here, not relevant to the topic.
...
}
]
},
"aggregations": {
"1": {
"value": 133
}
}
}
But if erase the log trace option, aggregations don't show up in the result :
[ { _index: 'worklight__appsession__1485302400000',
_type: 'AppSession',
... Some Data,
{ _index: 'worklight__appsession__1485302400000',
_type: 'AppSession',
... Some Data,
{ _index: 'worklight__appsession__1485302400000',
_type: 'AppSession',
... Some Data,
]
Am I doing something wrong, or do I just lack knowledge ?
Thanks for your time.
You are doing console.log(resp.hits.hits). Try this instead:
console.log(resp.aggregations)

Resources