Related
I'm currently facing an issue with my datatable implemented in ReactJS. I'm retrieving data from elasticsearch and populating the datatable with it. The data retrieval process works fine without the filter applied, however, when I apply filters to the data, the datatable remains empty, even though the data _source has matching records.
The structure of the parameters I am sending is as follows:
{
pageIndex: 1,
pageSize: 10,
sort: { order: '', key: '' },
query: '',
filterData: {
analysis: [ '0', '1', '2', '3' ],
threat_level_id: [ '1', '2', '3', '4' ],
}
}
EndPoint:
POST /api/v1/events/public/list
Controller:
exports.getPublicEvents = async (req, res) => {
try {
client.ping()
const { pageIndex, pageSize, sort, query, filterData } = req.body
let esQuery = {
index: 'ns_*',
body: {
query: {
bool: {
must: [
{
match_all: {},
},
],
filter: [],
},
},
from: (pageIndex - 1) * pageSize,
size: pageSize,
},
}
if (query) {
esQuery.body.query.bool.must = [
{
match: {
'Event.info': {
query: query,
fuzziness: 'AUTO',
},
},
},
]
}
if (filterData.analysis.length > 0) {
esQuery.body.query.bool.filter.push({
terms: {
'Event.analysis': filterData.analysis,
},
})
}
if (filterData.threat_level_id.length > 0) {
esQuery.body.query.bool.filter.push({
terms: {
'Event.threat_level_id': filterData.threat_level_id,
},
})
}
let esResponse = await client.search(esQuery)
let data = esResponse.hits.hits.map((hit) => hit._source)
let total = esResponse.hits.total.value
res.status(200).json({
status: 'success',
data: data,
total: total,
})
} catch (error) {
res.status(500).json({
error: 'Error connecting to Elasticsearch',
errorMessage: error.message,
})
}
}
The controller below is without filters and it works just fine.
exports.getPublicEvents = async (req, res) => {
try {
client.ping()
const { pageIndex, pageSize, sort, query } = req.body
let esQuery = {
index: 'ns_*',
body: {
query: {
match_all: {},
},
from: (pageIndex - 1) * pageSize,
size: pageSize,
},
}
if (query) {
esQuery.body.query = {
match: {
'Event.info': {
query: query,
fuzziness: 'AUTO',
},
},
}
}
let esResponse = await client.search(esQuery)
let data = esResponse.hits.hits.map((hit) => hit._source)
let total = esResponse.hits.total.value
res.status(200).json({
status: 'success',
data: data,
total: total,
})
} catch (error) {
res.status(500).json({
error: 'Error connecting to Elasticsearch',
errorMessage: error.message,
})
}
}
ElasticSearech version: 7.17.8
Result of: console.log(JSON.stringify(esQuery))
{
"index": "INDEX_NAME",
"body": {
"query": {
"bool": {
"must": [{ "match_all": {} }],
"filter": [
{ "terms": { "Event.analysis": ["0", "1", "2"] } },
{ "terms": { "Event.threat_level_id": ["1", "2", "3", "4"] } }
]
}
},
"from": 0,
"size": 10
}
}
Data in elascticsearch schema
{
"#version": "1",
"#timestamp": "2023-02-01T14:43:09.997Z",
"Event": {
"info": ".......................",
"description": ".......................",
"analysis": 0,
"threat_level_id": "4",
"created_at": 1516566351,
"uuid": "5a64f74f0e543738c12bc973322",
"updated_at": 1675262417
}
}
Index Mapping
{
"index_patterns": ["INDEX_NAME"],
"template": "TEMPLATE_NAME",
"settings": {
"number_of_replicas": 0,
"index.mapping.nested_objects.limit": 10000000
},
"mappings": {
"dynamic": false,
"properties": {
"#timestamp": {
"type": "date"
},
"Event": {
"type": "nested",
"properties": {
"date_occured": {
"type": "date"
},
"threat_level_id": {
"type": "integer"
},
"description": {
"type": "text"
},
"is_shared": {
"type": "boolean"
},
"analysis": {
"type": "integer"
},
"uuid": {
"type": "text"
},
"created_at": {
"type": "date"
},
"info": {
"type": "text"
},
"shared_with": {
"type": "nested",
"properties": {
"_id": {
"type": "text"
}
}
},
"updated_at": {
"type": "date"
},
"author": {
"type": "text"
},
"Attributes": {
"type": "nested",
"properties": {
"data": {
"type": "text"
},
"type": {
"type": "text"
},
"uuid": {
"type": "text"
},
"comment": {
"type": "text"
},
"category": {
"type": "text"
},
"value": {
"type": "text"
},
"timestamp": {
"type": "date"
}
}
},
"organisation": {
"type": "nested",
"properties": {
"name": {
"type": "text"
},
"uuid": {
"type": "text"
}
}
},
"Tags": {
"type": "nested",
"properties": {
"color": {
"type": "text"
},
"name": {
"type": "text"
}
}
},
"TLP": {
"type": "nested",
"properties": {
"color": {
"type": "text"
},
"name": {
"type": "text"
}
}
}
}
}
}
}
}
Event is a nested field, so you need to use nested queries, like this:
{
"index": "INDEX_NAME",
"body": {
"query": {
"bool": {
"must": [{ "match_all": {} }],
"filter": [
{
"nested": {
"path": "Event",
"query": {"terms": { "Event.analysis": ["0", "1", "2"] }}
}
},
{
"nested": {
"path": "Event",
"query": {"terms": { "Event.threat_level_id": ["1", "2", "3", "4"] }}
}
}
]
}
},
"from": 0,
"size": 10
}
}
I'm getting started with Google Analytics 4, and I have a task is creating chart report with custom dimensions. I have a problem when get data from API. It missing date data when getting data from GA4. I want to use this data to build a chart
const {
propertyId,
startDate,
endDate,
} = req.query;
// Request report data.
const analyticsDataClient = await getAnalyticsDataClient(getActiveShop(req));
const response = await analyticsDataClient.runPivotReport({
property: `properties/${propertyId}`,
dimensions: [
{ name: "date" },
{ name: "CUSTOM_DIMENSION_EVENT" },
{ name: "CUSTOM_DIMENSION_EVENT" },
{ name: "CUSTOM_DIMENSION_EVENT" },
],
metrics: [
{ name: "sessions" },
{ name: "activeUsers" },
{ name: "CUSTOM_METRIC_EVENT" },
{ name: "CUSTOM_METRIC_EVENT" },
],
dateRanges: [{ startDate: 2022-12-06, endDate: 2023-01-03 }], // Dateranges
pivots: [
{
fieldNames: ["date"],
limit: 30,
},
{
fieldNames: ["CUSTOM_DIMENSION_EVENT"],
limit: 5,
},
{
fieldNames: ["CUSTOM_DIMENSION_EVENT"],
limit: 5,
},
{
fieldNames: ["CUSTOM_DIMENSION_EVENT"],
limit: 5,
},
],
dimensionFilter: {
filter: {
fieldName: "CUSTOM_METRIC_EVENT", // id
stringFilter: {
matchType: "EXACT",
value: "CUSTOM_METRIC_EVENT_ID",
},
},
},
keepEmptyRows: true,
});
response GA4 API
{
"response": [
{
"pivotHeaders": [
{
"pivotDimensionHeaders": [
{
"dimensionValues": [
{
"value": "20221227", // return this date only
"oneValue": "value"
}
]
}
],
"rowCount": 1
},
{
"pivotDimensionHeaders": [
{
"dimensionValues": [
{
"value": "885c7b0d-bc65-47be-b7df-871947bc5de4",
"oneValue": "value"
}
]
}
],
"rowCount": 1
},
{
"pivotDimensionHeaders": [
{
"dimensionValues": [
{
"value": "New Page For Test GA4",
"oneValue": "value"
}
]
}
],
"rowCount": 1
},
{
"pivotDimensionHeaders": [
{
"dimensionValues": [
{
"value": "index",
"oneValue": "value"
}
]
}
],
"rowCount": 1
}
],
"dimensionHeaders": [
{
"name": "date"
},
{
"name": "customEvent:page_id"
},
{
"name": "customEvent:page_name"
},
{
"name": "customEvent:page_type"
}
],
"metricHeaders": [
{
"name": "sessions",
"type": "TYPE_INTEGER"
},
{
"name": "activeUsers",
"type": "TYPE_INTEGER"
},
{
"name": "customEvent:revenue",
"type": "TYPE_CURRENCY"
}
],
"rows": [
{
"dimensionValues": [
{
"value": "20221227",
"oneValue": "value"
},
{
"value": "885c7b0d-bc65-47be-b7df-871947bc5de4",
"oneValue": "value"
},
{
"value": "New Page For Test GA4",
"oneValue": "value"
},
{
"value": "index",
"oneValue": "value"
}
],
"metricValues": [
{
"value": "7",
"oneValue": "value"
},
{
"value": "7",
"oneValue": "value"
},
{
"value": "0",
"oneValue": "value"
},
{
"value": "8",
"oneValue": "value"
}
]
}
],
"aggregates": [],
"metadata": {
"dataLossFromOtherRow": false,
"schemaRestrictionResponse": {
"activeMetricRestrictions": []
},
"_schemaRestrictionResponse": "schemaRestrictionResponse",
"_currencyCode": "currencyCode",
"timeZone": "America/Los_Angeles",
"_timeZone": "timeZone"
},
"propertyQuota": null,
"kind": "analyticsData#runPivotReport"
},
null,
null
]
}
Although I set dateRanges { startDate: 2022-12-06, endDate: 2023-01-03 } , it only returns 20221227 for me? What's problem here?
If you check runPivotReport in the doucmentation you will find that the response body for a RunPivotReportResponse
Does not contain dates, the api probably assumes you know what dates you requested.
{
"pivotHeaders": [
{
object (PivotHeader)
}
],
"dimensionHeaders": [
{
object (DimensionHeader)
}
],
"metricHeaders": [
{
object (MetricHeader)
}
],
"rows": [
{
object (Row)
}
],
"aggregates": [
{
object (Row)
}
],
"metadata": {
object (ResponseMetaData)
},
"propertyQuota": {
object (PropertyQuota)
},
"kind": string
}
I'm having trouble getting and updating the only document that matches filter in nest array of objects in mongoose, I'm using the findOneAndUpdate query in mongoose.
This is my data:
{
"_id": "62e87e193fe01f5068f9ae11",
"year": "2023",
"month": "1",
"department_id":"62e387d39ffb6ada6c590fbf",
"blocks": [
{
"name": "CEEDO Schedule Block",
"days": [
{
"day": 2,
"employees": [
{
"employee_id":"62cf92fb3a790000170062e3",
"schedule_type": "Day Off"
},
{
"employee_id": "62cf92fb3a790000170062e2",
"schedule_type": "Shifting"
},
{
"employee_id": "62cf92fb3a790000170062e4",
"schedule_type": "Regular"
}
],
"_id": "62e87e193fe01f5068f9ae13"
},
{
"day": 6,
"employees": [
{
"employee_id": "62cf92fb3a790000170062e3",
"schedule_type": "Day Off"
},
{
"employee_id": "62cf92fb3a790000170062e2",
"schedule_type": "Shifting"
},
{
"employee_id":"62cf92fb3a790000170062e4",
"schedule_type": "Regular"
}
],
"_id": "62e87e193fe01f5068f9ae14"
}
],
"_id": "62e87e193fe01f5068f9ae12"
}
]
}
And here is my query:
const update_block = await schedule_model.findOneAndUpdate({'blocks.days._id': '62e87e193fe01f5068f9ae13'},
{
$set: {"days":req.body.days, "employees":req.body.employees}
}
);
Thanks in advance.
try change '62e87e193fe01f5068f9ae13' to mongoose.Types.ObjectId('62e87e193fe01f5068f9ae13')
I finally found the answer by using the arrayFilter function in mongoose:
const update_block = await schedule_model.updateOne({
"_id": mongoose.Types.ObjectId('62e87e193fe01f5068f9ae11')
}, {
"$set": {
"blocks.$[i].days.$[j].day": 31
}
}, {
arrayFilters: [{
"i._id":mongoose.Types.ObjectId('62e87e193fe01f5068f9ae12')
}, {
"j._id": mongoose.Types.ObjectId('62e87e193fe01f5068f9ae14')
}]
})
console.log(update_block)
Thank you.
I'm trying to filter a nested object and sort by the result, however, I tried some things without success, I'll leave my initial attempt and it works partially, it just filters according to what I have in my search variable, but all the results come of this nested object as it is inside the 'root' which is another nested object
Elastic version: 7.13.0 with NodeJS
using #elastic/elasticsearch official package from npm
let params: RequestParams.Search = {
index: index,
body: {
size: 30,
query: {
bool: {
must: [
{
nested: {
path: "profile",
query: {
bool: {
must: [
{
match: {
"profile.id": profileId,
},
},
],
},
},
},
},
],
filter: [
{
nested: {
path: "profile.following",
ignore_unmapped: true,
query: {
query_string: {
fields: [
"profile.following.name",
"profile.following.username",
],
query: searchWord + "*",
},
},
},
},
],
},
},
},
};
I need it to be this specific 'profile.id' that is passed by parameter in the function, so the result is only 1 profile with N people that it follows
the document is mapped as follows, I left only the fields relevant to the question:
{
"mappings": {
"_doc": {
"properties": {
"id": {
"type": "integer"
},
"phone": {
"type": "text"
},
"profile": {
"type": "nested",
"properties": {
"id": {
"type": "integer"
},
"username": {
"type": "text"
},
"following": {
"type": "nested",
"properties": {
"id": {
"type": "integer"
},
"isAwaitingApproval": {
"type": "boolean"
},
"name": {
"type": "text"
},
"profilePicURL": {
"type": "text"
},
"username": {
"type": "text"
}
}
}
}
}
}
}
}
}
an example of a current result is:
with the following parameters (profileId:141, searchWord: "para" )
{
"res": [
{
"profilePicURL": "localimage",
"name": "donor donor",
"id": 140,
"username": "victorTesteElastic2",
"isAwaitingApproval": false
},
{
"profilePicURL": "localimage",
"name": "para ser seguido",
"id": 142,
"username": "victorprivate",
"isAwaitingApproval": true
}
]
}
the desired result is:
{
"res": [
{
"profilePicURL": "localimage",
"name": "para ser seguido",
"id": 142,
"username": "victorprivate",
"isAwaitingApproval": true
}
]
}
with some more research I got what I needed, I'll leave the answer here in case anyone needs it too
let params: RequestParams.Search = {
index: index,
body: {
size: 30,
query: {
bool: {
must: [
{
nested: {
path: "profile",
query: {
bool: {
must: [
{
match: {
"profile.id": profileId,
},
},
],
},
},
},
},
{
nested: {
path: "profile",
inner_hits: {
name: "profile",
},
query: {
nested: {
path: "profile.following",
inner_hits: {
name: "following",
},
ignore_unmapped: true,
query: {
query_string: {
fields: [
"profile.following.name",
"profile.following.username",
],
query: searchWord + "*",
},
},
},
},
},
},
],
},
},
},
};
I basically put in must what was in the filter, mapped the nested object from above, in this case the profile, and put the tag inner_hits for profile and inner_hits for followings, that's the only way it worked
the answer I need was returned here:
body.hits.hits[0].inner_hits.profile.hits.hits[0].inner_hits.following.hits.hits
below is an example of the answer:
{
"res": [
{
"_index": "donor",
"_type": "_doc",
"_id": "P3VWNnsB4coAEhD-F3fF",
"_nested": {
"field": "profile",
"offset": 0,
"_nested": {
"field": "following",
"offset": 0
}
},
"_score": 1,
"_source": {
"profilePicURL": "localimage",
"name": "donor donor",
"id": 140,
"username": "victorTesteElastic2",
"isAwaitingApproval": false
}
},
{
"_index": "donor",
"_type": "_doc",
"_id": "P3VWNnsB4coAEhD-F3fF",
"_nested": {
"field": "profile",
"offset": 0,
"_nested": {
"field": "following",
"offset": 1
}
},
"_score": 1,
"_source": {
"profilePicURL": "localimage",
"name": "para ser seguido",
"id": 142,
"username": "victorprivate",
"isAwaitingApproval": true
}
}
]
}
the filtered data I really need that have been matched in must is in this array, where I need to iterate and look at _source which is the data that is indexed
Following is the Sample Workspace document.I want to update box positions when we drag and drop at front end.
{
"_id": ObjectId("5eaa9b7c87e99ef2430a320b"),
"logo": {
"url": ".../../../assets/logo/dsdsds.png",
"name": "testUpload"
},
"name": "My World",
"sections": [{
"box": [{
"_id": ObjectId("5da87b33502d6c634b3aa7ce"),
"name": "Meran To",
"position": 0
},
{
"_id": ObjectId("5da87b33502d6c7d873aa7d0"),
"name": "Documentation",
"position": 2
},
{
"_id": ObjectId("5da87b33502d6cdbb93aa7cf"),
"name": "File Manager Upload File Drive",
"position": 1
},
{
"_id": ObjectId("5da87b33502d6c276a3aa7cd"),
"name": "File Manager Upload File Drive",
"position": 1
}
],
"id": 1,
"title": "Simplicity",
"description": "Follow your barriers"
},
{
"box": [],
"id": 2,
"title": "xfxdfxcx 34",
"description": "sdsdsd sfsfsd ewrewrewre"
}
]
}
I send the updated positions from front-end to back-end via API, in an array as shown below.
[
{
"id": "5da87b33502d6c634b3aa7ce",
"position": 0
}, {
"id": "5da87b33502d6c7d873aa7d0",
"position": 1
}, {
"id": "5da87b33502d6cdbb93aa7cf",
"position": 2
}, {
"id": "5da87b33502d6c276a3aa7cd",
"position": 3
}]
I am currently updating DB using the below code
for (const el of req.body) {
await this.model.updateOne({
_id: req.params.workspaceId,
sections: {
$elemMatch: {
id: req.params.sectionId
}
},
'sections.box': {
$elemMatch: {
_id: el.id
}
},
}, {
$set: {
'sections.$[outer].box.$[inner].position': el.position
}
}, {
arrayFilters: [{
'outer.id': req.params.sectionId
}, {
'inner._id': el.id
}],
upsert: false,
});
}
But this is not the best method, it hits DB multiple times.
so I need to optimize this code with mongoose query itself.
May be using $set / $push.I don't know any exact methods.
So basically we need to remove the external for loop and make it work with mongoose itself.This is my requirement.
Thanks in advance for all the support.
There are 2 methods for doing it.
bulkWrite
const bulkOps = [];
req.body.forEach((el) => {
const upsertDoc = {
updateOne: {
filter: {
_id: req.params.workspaceId,
sections: {
$elemMatch: {
id: req.params.sectionId
}
},
'sections.box': {
$elemMatch: {
_id: el.id
}
},
},
update: {
$set: {
'sections.$[outer].box.$[inner].position': el.position
}
},
arrayFilters: [{
'outer.id': req.params.sectionId
}, {
'inner._id': el.id
}],
upsert: false,
}
};
bulkOps.push(upsertDoc);
});
const result = await this.model.collection.bulkWrite(bulkOps);
bulkUpdate
const bulkUpdate = this.model.collection.initializeUnorderedBulkOp();
req.body.forEach((el) => {
bulkUpdate.find({
_id: req.params.workspaceId,
sections: { $elemMatch: { id: req.params.sectionId } },
'sections.box': { $elemMatch: { _id: el.id } },
}).arrayFilters([{ 'outer.id': req.params.sectionId }, { 'inner._id': el.id }])
.updateOne({ $set: { 'sections.$[outer].box.$[inner].position': el.position }
});
});
await bulkUpdate.execute();