Related
I'm currently facing an issue with my datatable implemented in ReactJS. I'm retrieving data from elasticsearch and populating the datatable with it. The data retrieval process works fine without the filter applied, however, when I apply filters to the data, the datatable remains empty, even though the data _source has matching records.
The structure of the parameters I am sending is as follows:
{
pageIndex: 1,
pageSize: 10,
sort: { order: '', key: '' },
query: '',
filterData: {
analysis: [ '0', '1', '2', '3' ],
threat_level_id: [ '1', '2', '3', '4' ],
}
}
EndPoint:
POST /api/v1/events/public/list
Controller:
exports.getPublicEvents = async (req, res) => {
try {
client.ping()
const { pageIndex, pageSize, sort, query, filterData } = req.body
let esQuery = {
index: 'ns_*',
body: {
query: {
bool: {
must: [
{
match_all: {},
},
],
filter: [],
},
},
from: (pageIndex - 1) * pageSize,
size: pageSize,
},
}
if (query) {
esQuery.body.query.bool.must = [
{
match: {
'Event.info': {
query: query,
fuzziness: 'AUTO',
},
},
},
]
}
if (filterData.analysis.length > 0) {
esQuery.body.query.bool.filter.push({
terms: {
'Event.analysis': filterData.analysis,
},
})
}
if (filterData.threat_level_id.length > 0) {
esQuery.body.query.bool.filter.push({
terms: {
'Event.threat_level_id': filterData.threat_level_id,
},
})
}
let esResponse = await client.search(esQuery)
let data = esResponse.hits.hits.map((hit) => hit._source)
let total = esResponse.hits.total.value
res.status(200).json({
status: 'success',
data: data,
total: total,
})
} catch (error) {
res.status(500).json({
error: 'Error connecting to Elasticsearch',
errorMessage: error.message,
})
}
}
The controller below is without filters and it works just fine.
exports.getPublicEvents = async (req, res) => {
try {
client.ping()
const { pageIndex, pageSize, sort, query } = req.body
let esQuery = {
index: 'ns_*',
body: {
query: {
match_all: {},
},
from: (pageIndex - 1) * pageSize,
size: pageSize,
},
}
if (query) {
esQuery.body.query = {
match: {
'Event.info': {
query: query,
fuzziness: 'AUTO',
},
},
}
}
let esResponse = await client.search(esQuery)
let data = esResponse.hits.hits.map((hit) => hit._source)
let total = esResponse.hits.total.value
res.status(200).json({
status: 'success',
data: data,
total: total,
})
} catch (error) {
res.status(500).json({
error: 'Error connecting to Elasticsearch',
errorMessage: error.message,
})
}
}
ElasticSearech version: 7.17.8
Result of: console.log(JSON.stringify(esQuery))
{
"index": "INDEX_NAME",
"body": {
"query": {
"bool": {
"must": [{ "match_all": {} }],
"filter": [
{ "terms": { "Event.analysis": ["0", "1", "2"] } },
{ "terms": { "Event.threat_level_id": ["1", "2", "3", "4"] } }
]
}
},
"from": 0,
"size": 10
}
}
Data in elascticsearch schema
{
"#version": "1",
"#timestamp": "2023-02-01T14:43:09.997Z",
"Event": {
"info": ".......................",
"description": ".......................",
"analysis": 0,
"threat_level_id": "4",
"created_at": 1516566351,
"uuid": "5a64f74f0e543738c12bc973322",
"updated_at": 1675262417
}
}
Index Mapping
{
"index_patterns": ["INDEX_NAME"],
"template": "TEMPLATE_NAME",
"settings": {
"number_of_replicas": 0,
"index.mapping.nested_objects.limit": 10000000
},
"mappings": {
"dynamic": false,
"properties": {
"#timestamp": {
"type": "date"
},
"Event": {
"type": "nested",
"properties": {
"date_occured": {
"type": "date"
},
"threat_level_id": {
"type": "integer"
},
"description": {
"type": "text"
},
"is_shared": {
"type": "boolean"
},
"analysis": {
"type": "integer"
},
"uuid": {
"type": "text"
},
"created_at": {
"type": "date"
},
"info": {
"type": "text"
},
"shared_with": {
"type": "nested",
"properties": {
"_id": {
"type": "text"
}
}
},
"updated_at": {
"type": "date"
},
"author": {
"type": "text"
},
"Attributes": {
"type": "nested",
"properties": {
"data": {
"type": "text"
},
"type": {
"type": "text"
},
"uuid": {
"type": "text"
},
"comment": {
"type": "text"
},
"category": {
"type": "text"
},
"value": {
"type": "text"
},
"timestamp": {
"type": "date"
}
}
},
"organisation": {
"type": "nested",
"properties": {
"name": {
"type": "text"
},
"uuid": {
"type": "text"
}
}
},
"Tags": {
"type": "nested",
"properties": {
"color": {
"type": "text"
},
"name": {
"type": "text"
}
}
},
"TLP": {
"type": "nested",
"properties": {
"color": {
"type": "text"
},
"name": {
"type": "text"
}
}
}
}
}
}
}
}
Event is a nested field, so you need to use nested queries, like this:
{
"index": "INDEX_NAME",
"body": {
"query": {
"bool": {
"must": [{ "match_all": {} }],
"filter": [
{
"nested": {
"path": "Event",
"query": {"terms": { "Event.analysis": ["0", "1", "2"] }}
}
},
{
"nested": {
"path": "Event",
"query": {"terms": { "Event.threat_level_id": ["1", "2", "3", "4"] }}
}
}
]
}
},
"from": 0,
"size": 10
}
}
I'm getting started with Google Analytics 4, and I have a task is creating chart report with custom dimensions. I have a problem when get data from API. It missing date data when getting data from GA4. I want to use this data to build a chart
const {
propertyId,
startDate,
endDate,
} = req.query;
// Request report data.
const analyticsDataClient = await getAnalyticsDataClient(getActiveShop(req));
const response = await analyticsDataClient.runPivotReport({
property: `properties/${propertyId}`,
dimensions: [
{ name: "date" },
{ name: "CUSTOM_DIMENSION_EVENT" },
{ name: "CUSTOM_DIMENSION_EVENT" },
{ name: "CUSTOM_DIMENSION_EVENT" },
],
metrics: [
{ name: "sessions" },
{ name: "activeUsers" },
{ name: "CUSTOM_METRIC_EVENT" },
{ name: "CUSTOM_METRIC_EVENT" },
],
dateRanges: [{ startDate: 2022-12-06, endDate: 2023-01-03 }], // Dateranges
pivots: [
{
fieldNames: ["date"],
limit: 30,
},
{
fieldNames: ["CUSTOM_DIMENSION_EVENT"],
limit: 5,
},
{
fieldNames: ["CUSTOM_DIMENSION_EVENT"],
limit: 5,
},
{
fieldNames: ["CUSTOM_DIMENSION_EVENT"],
limit: 5,
},
],
dimensionFilter: {
filter: {
fieldName: "CUSTOM_METRIC_EVENT", // id
stringFilter: {
matchType: "EXACT",
value: "CUSTOM_METRIC_EVENT_ID",
},
},
},
keepEmptyRows: true,
});
response GA4 API
{
"response": [
{
"pivotHeaders": [
{
"pivotDimensionHeaders": [
{
"dimensionValues": [
{
"value": "20221227", // return this date only
"oneValue": "value"
}
]
}
],
"rowCount": 1
},
{
"pivotDimensionHeaders": [
{
"dimensionValues": [
{
"value": "885c7b0d-bc65-47be-b7df-871947bc5de4",
"oneValue": "value"
}
]
}
],
"rowCount": 1
},
{
"pivotDimensionHeaders": [
{
"dimensionValues": [
{
"value": "New Page For Test GA4",
"oneValue": "value"
}
]
}
],
"rowCount": 1
},
{
"pivotDimensionHeaders": [
{
"dimensionValues": [
{
"value": "index",
"oneValue": "value"
}
]
}
],
"rowCount": 1
}
],
"dimensionHeaders": [
{
"name": "date"
},
{
"name": "customEvent:page_id"
},
{
"name": "customEvent:page_name"
},
{
"name": "customEvent:page_type"
}
],
"metricHeaders": [
{
"name": "sessions",
"type": "TYPE_INTEGER"
},
{
"name": "activeUsers",
"type": "TYPE_INTEGER"
},
{
"name": "customEvent:revenue",
"type": "TYPE_CURRENCY"
}
],
"rows": [
{
"dimensionValues": [
{
"value": "20221227",
"oneValue": "value"
},
{
"value": "885c7b0d-bc65-47be-b7df-871947bc5de4",
"oneValue": "value"
},
{
"value": "New Page For Test GA4",
"oneValue": "value"
},
{
"value": "index",
"oneValue": "value"
}
],
"metricValues": [
{
"value": "7",
"oneValue": "value"
},
{
"value": "7",
"oneValue": "value"
},
{
"value": "0",
"oneValue": "value"
},
{
"value": "8",
"oneValue": "value"
}
]
}
],
"aggregates": [],
"metadata": {
"dataLossFromOtherRow": false,
"schemaRestrictionResponse": {
"activeMetricRestrictions": []
},
"_schemaRestrictionResponse": "schemaRestrictionResponse",
"_currencyCode": "currencyCode",
"timeZone": "America/Los_Angeles",
"_timeZone": "timeZone"
},
"propertyQuota": null,
"kind": "analyticsData#runPivotReport"
},
null,
null
]
}
Although I set dateRanges { startDate: 2022-12-06, endDate: 2023-01-03 } , it only returns 20221227 for me? What's problem here?
If you check runPivotReport in the doucmentation you will find that the response body for a RunPivotReportResponse
Does not contain dates, the api probably assumes you know what dates you requested.
{
"pivotHeaders": [
{
object (PivotHeader)
}
],
"dimensionHeaders": [
{
object (DimensionHeader)
}
],
"metricHeaders": [
{
object (MetricHeader)
}
],
"rows": [
{
object (Row)
}
],
"aggregates": [
{
object (Row)
}
],
"metadata": {
object (ResponseMetaData)
},
"propertyQuota": {
object (PropertyQuota)
},
"kind": string
}
hello I am trying to update a document on mongodb but everytime I run the lambda function below it does not return the new document but returns a http response 200. I have tried to update the logic in so many forms but I get the response 200 but the update does not happen. The lambda function takes 2 path parameters i.e the userid and the date to find the specific plan then updates it... It neither updates nor returns the new document. Please help!
const Plan = require('../../model/planModel');
const User = require('../../model/userModel');
const { connectDB } = require('../../config/db');
const { verify, decode } = require('jsonwebtoken');
const { decode_token } = require('../../utils');
const { findByIdAndUpdate } = require('../../model/planModel');
exports.handler = async (event) => {
try {
await connectDB();
//console.log(event)
const { userid, date } = event.pathParameters;
const exists = await Plan.findOne(userid, date);
console.log('I made it here: ', exists);
const {
excitedAbout,
gratefulForMorning,
mainThreeTodo,
dailyToDo,
meals,
water,
exercise,
meditation,
relaxation,
happyScale,
happyMoment,
gratefulForTonight,
dailyQuestion,
} = event.body;
const updatedPlan = await Plan.findByIdAndUpdate(
{ _id: exists._id },
{
excitedAbout: excitedAbout,
gratefulForMorning: gratefulForMorning,
mainThreeTodo: mainThreeTodo,
dailyToDo: dailyToDo,
meals: meals,
water: water,
exercise: exercise,
meditation: meditation,
relaxation: relaxation,
happyScale: happyScale,
happyMoment: happyMoment,
gratefulForTonight: gratefulForTonight,
dailyQuestion: dailyQuestion,
},
{ new: true }
);
console.log('this is a plan', updatedPlan);
return {
statusCode: 200,
headers: {
'Access-Control-Allow-Origin': '*',
},
body: JSON.stringify(updatedPlan),
};
} catch (error) {
console.log(error);
}
};
This is the data I want to update:
{
"user": "62d1a251806d10095ff0f0a6",
"date": "2022-09-09T18:30:00.000Z",
"excitedAbout": "Excited About something",
"gratefulForMorning": "",
"mainThreeTodo": [
{
"text": "HELLO I survived",
"checked": false
},
{
"text": "",
"checked": false
},
{
"text": "",
"checked": false
}
],
"dailyToDo": [
{
"text": "",
"checked": false
},
{
"text": "",
"checked": false
},
{
"text": "",
"checked": false
}
],
"meals": {
"breakfast": [
{
"meal": "",
"calories": ""
}
],
"lunch": [
{
"meal": "",
"calories": ""
}
],
"dinner": [
{
"meal": "skuma",
"calories": ""
}
]
},
"water": 7,
"exercise": false,
"relaxation": false,
"meditation": false,
"happyScale": 0,
"happyMoment": "",
"gratefulForTonight": "",
"dailyQuestion": {
"question": "",
"answer": ""
}
}
and this is what I'm getting
{
"meals": {
"breakfast": [
{
"meal": "",
"calories": "",
"_id": "631cc36ed9e6f6711e7da2c3"
}
],
"lunch": [
{
"meal": "",
"calories": "",
"_id": "631cc36ed9e6f6711e7da2c2"
}
],
"dinner": [
{
"meal": "",
"calories": "",
"_id": "631cc36ed9e6f6711e7da2c1"
}
]
},
"dailyQuestion": {
"question": "",
"answer": ""
},
"_id": "631ba3e559b8a53d732a1958",
"user": "62d1a251806d10095ff0f0a6",
"date": "2022-09-09T18:30:00.000Z",
"excitedAbout": "",
"gratefulForMorning": "",
"mainThreeTodo": [
{
"text": "",
"checked": false,
"_id": "631cc36ed9e6f6711e7da2bb"
},
{
"text": "",
"checked": false,
"_id": "631cc36ed9e6f6711e7da2bc"
},
{
"text": "",
"checked": false,
"_id": "631cc36ed9e6f6711e7da2bd"
}
],
"dailyToDo": [
{
"text": "",
"checked": false,
"_id": "631cc36ed9e6f6711e7da2be"
},
{
"text": "",
"checked": false,
"_id": "631cc36ed9e6f6711e7da2bf"
},
{
"text": "",
"checked": false,
"_id": "631cc36ed9e6f6711e7da2c0"
}
],
"water": 0,
"exercise": false,
"meditation": false,
"relaxation": false,
"happyScale": 0,
"happyMoment": "",
"gratefulForTonight": "",
"createdAt": "2022-09-09T20:36:53.680Z",
"updatedAt": "2022-09-13T12:01:02.966Z",
"__v": 0
}
it's like it is never passed
Your query seems to be missing the update operators, that's why it might not be working for you:
const updatedPlan = await Plan.findByIdAndUpdate(
{ _id: exists._id },
{
$set: {
excitedAbout: excitedAbout,
gratefulForMorning: gratefulForMorning,
mainThreeTodo: mainThreeTodo,
dailyToDo: dailyToDo,
meals: meals,
water: water,
exercise: exercise,
meditation: meditation,
relaxation: relaxation,
happyScale: happyScale,
happyMoment: happyMoment,
gratefulForTonight: gratefulForTonight,
dailyQuestion: dailyQuestion,
}
},
{ new: true }
);
Try this, here we have specified the $set update operator.
I'm trying to filter a nested object and sort by the result, however, I tried some things without success, I'll leave my initial attempt and it works partially, it just filters according to what I have in my search variable, but all the results come of this nested object as it is inside the 'root' which is another nested object
Elastic version: 7.13.0 with NodeJS
using #elastic/elasticsearch official package from npm
let params: RequestParams.Search = {
index: index,
body: {
size: 30,
query: {
bool: {
must: [
{
nested: {
path: "profile",
query: {
bool: {
must: [
{
match: {
"profile.id": profileId,
},
},
],
},
},
},
},
],
filter: [
{
nested: {
path: "profile.following",
ignore_unmapped: true,
query: {
query_string: {
fields: [
"profile.following.name",
"profile.following.username",
],
query: searchWord + "*",
},
},
},
},
],
},
},
},
};
I need it to be this specific 'profile.id' that is passed by parameter in the function, so the result is only 1 profile with N people that it follows
the document is mapped as follows, I left only the fields relevant to the question:
{
"mappings": {
"_doc": {
"properties": {
"id": {
"type": "integer"
},
"phone": {
"type": "text"
},
"profile": {
"type": "nested",
"properties": {
"id": {
"type": "integer"
},
"username": {
"type": "text"
},
"following": {
"type": "nested",
"properties": {
"id": {
"type": "integer"
},
"isAwaitingApproval": {
"type": "boolean"
},
"name": {
"type": "text"
},
"profilePicURL": {
"type": "text"
},
"username": {
"type": "text"
}
}
}
}
}
}
}
}
}
an example of a current result is:
with the following parameters (profileId:141, searchWord: "para" )
{
"res": [
{
"profilePicURL": "localimage",
"name": "donor donor",
"id": 140,
"username": "victorTesteElastic2",
"isAwaitingApproval": false
},
{
"profilePicURL": "localimage",
"name": "para ser seguido",
"id": 142,
"username": "victorprivate",
"isAwaitingApproval": true
}
]
}
the desired result is:
{
"res": [
{
"profilePicURL": "localimage",
"name": "para ser seguido",
"id": 142,
"username": "victorprivate",
"isAwaitingApproval": true
}
]
}
with some more research I got what I needed, I'll leave the answer here in case anyone needs it too
let params: RequestParams.Search = {
index: index,
body: {
size: 30,
query: {
bool: {
must: [
{
nested: {
path: "profile",
query: {
bool: {
must: [
{
match: {
"profile.id": profileId,
},
},
],
},
},
},
},
{
nested: {
path: "profile",
inner_hits: {
name: "profile",
},
query: {
nested: {
path: "profile.following",
inner_hits: {
name: "following",
},
ignore_unmapped: true,
query: {
query_string: {
fields: [
"profile.following.name",
"profile.following.username",
],
query: searchWord + "*",
},
},
},
},
},
},
],
},
},
},
};
I basically put in must what was in the filter, mapped the nested object from above, in this case the profile, and put the tag inner_hits for profile and inner_hits for followings, that's the only way it worked
the answer I need was returned here:
body.hits.hits[0].inner_hits.profile.hits.hits[0].inner_hits.following.hits.hits
below is an example of the answer:
{
"res": [
{
"_index": "donor",
"_type": "_doc",
"_id": "P3VWNnsB4coAEhD-F3fF",
"_nested": {
"field": "profile",
"offset": 0,
"_nested": {
"field": "following",
"offset": 0
}
},
"_score": 1,
"_source": {
"profilePicURL": "localimage",
"name": "donor donor",
"id": 140,
"username": "victorTesteElastic2",
"isAwaitingApproval": false
}
},
{
"_index": "donor",
"_type": "_doc",
"_id": "P3VWNnsB4coAEhD-F3fF",
"_nested": {
"field": "profile",
"offset": 0,
"_nested": {
"field": "following",
"offset": 1
}
},
"_score": 1,
"_source": {
"profilePicURL": "localimage",
"name": "para ser seguido",
"id": 142,
"username": "victorprivate",
"isAwaitingApproval": true
}
}
]
}
the filtered data I really need that have been matched in must is in this array, where I need to iterate and look at _source which is the data that is indexed
{
"success": true,
"message": "Result",
"data": [
{
"Here": [
{
"_id": "5ee97ee7f25d1c1482717bdf",
"email": "test1#test.io",
"profileImages": [],
"username": "test1",
"birthday": "2020-06-11T10:11:32.000Z",
"phoneNumber": "+910000000000",
"location": "Test Location",
"firstName": "test1",
"lastName": "test1",
}
]
},
{
"Here": [
{
"_id": "5ee97ef2f25d1c1482717be1",
"email": "test2#test.io",
"profileImages": [],
"username": "test2",
"birthday": "2020-06-11T10:11:32.000Z",
"phoneNumber": "+910000000000",
"location": "Test Location"
}
]
}
],
}
What I am expecting is this
{
"success": true,
"message": "Result",
data: [
{
"_id": "5ee97ee7f25d1c1482717bdf",
"email": "test1#test.io",
"profileImages": [],
"username": "test1",
"birthday": "2020-06-11T10:11:32.000Z",
"phoneNumber": "+910000000000",
"location": "Test Location",
"firstName": "test1",
"lastName": "test1"},
{
"_id": "5ee97ef2f25d1c1482717be1",
"email": "test2#test.io",
"profileImages": [],
"username": "test2",
"birthday": "2020-06-11T10:11:32.000Z",
"phoneNumber": "+910000000000",
"location": "Test Location"
}
]
}
Query I am using is for this response is below using aggregation in mongodb, lookup and project which is leading me to the some undesired response
db.collections.aggregate( [
{
$lookup: {
from: 'users',
as: 'Here',
let: {
whoDid: '$whoDid'
},
pipeline: [
{
"$match": { "$expr": { "$eq": ["$_id", "$$whoDid"] } }
},
{
$project: {
_id: 1,
email: 1,
profileImages: 1,
username: 1,
birthday: 1,
phoneNumber: 1,
firstName: 1,
lastName: 1,
fullName: 1,
// age: {$year: "$birthday"}
age: {
$divide: [{ $subtract: [new Date(), "$birthday"] },
(31558464000)]
}
}
}
],
}
},
{
$project:{
Here:1,
_id:0
}
} ,
])
who did table is one of the collection I have where I have stored the user Id and later I am populating the data using lookup
{
"_id" : ObjectId("5ee988eb1aac0022e15dbb7b"),
"whoDid" : ObjectId("5ee97ef2f25d1c1482717be1"),
"toWhomDid" : ObjectId("5ee97ec0f25d1c1482717bdd"),
"modified_at" : ISODate("2020-06-17T03:07:23.217Z"),
"created_at" : ISODate("2020-06-17T03:07:23.217Z"),
"__v" : 0
}
{
"_id" : ObjectId("5ee988eb1aac0022e15dbb7c"),
"whoDid" : ObjectId("5ee97ec0f25d1c1482717bdd"),
"toWhomDid" : ObjectId("5ee97ef2f25d1c1482717be1"),
"modified_at" : ISODate("2020-06-17T03:07:23.220Z"),
"created_at" : ISODate("2020-06-17T03:07:23.220Z"),
"__v" : 0
}
Can anyone suggest me any better option so that I can get a desired respose?
It is possible to use reduce method:
obj.data = obj.data.reduce((a, c) => {
a.push(...c.Here);
return a;
}, [])
An example:
let obj = {
"success": true,
"message": "Result",
"data": [ {
"Here": [ {
"_id": "5ee97ee7f25d1c1482717bdf", "email": "test1#test.io",
"profileImages": [], "username": "test1",
"birthday": "2020-06-11T10:11:32.000Z", "phoneNumber": "+910000000000", "location": "Test Location",
"firstName": "test1", "lastName": "test1",
}
]
},
{
"Here": [ {
"_id": "5ee97ef2f25d1c1482717be1",
"email": "test2#test.io",
"profileImages": [],
"username": "test2",
"birthday": "2020-06-11T10:11:32.000Z",
"phoneNumber": "+910000000000",
"location": "Test Location"
}
]
}
]
};
obj.data = obj.data.reduce((a, c) => {
a.push(...c.Here);
return a;
}, [])
console.log(obj);
Add these extra steps into your aggregation pipeline:
{
$unwind: "$Here"
},
{
$replaceWith: "$Here"
}
MongoPlayground
Note: You can replace $project: { _id: 1, email: 1, ... to this:
{
$addFields:{
age: {
$divide: [{ $subtract: [new Date(), "$birthday"] },(31558464000)]
}
}
}