How to fetch single matched document from nested array mongodb? - node.js

I have nested array object in my MongoDB document, but the predicate is not return as expected, which in this case I only want matched document only.
Document Structure
{
"_id": {
"$oid": "5fddb34848be35283c36955b"
},
"projectId": {
"$oid": "5fddb30d48be35283c36955a"
},
"urls": [
{
"_id": {
"$oid": "5fddb34848be35283c36955c"
},
"group": "User",
"data": [
{
"option": {
"applyDifferentContentType": false,
"allowInjection": false
},
"_id": {
"$oid": "5fddad1e48be35283c369558"
},
"url": "/users/create",
"method": "patch",
"headers": {
"Content-Type": "multipart/form-data"
}
},
{
"option": {
"applyDifferentContentType": false,
"allowInjection": false
},
"_id": {
"$oid": "5fddad1e48be35283c369558"
},
"url": "/users/update",
"method": "post",
"headers": {
"Content-Type": "application/json"
}
},
{
"option": {
"applyDifferentContentType": false,
"allowInjection": false
},
"_id": {
"$oid": "5fddad1e48be35283c369558"
},
"url": "/users/delete",
"method": "post",
"headers": {
"Content-Type": "application/json"
}
}
]
},
{
"_id": {
"$oid": "5fddb34f48be35283c36955d"
},
"group": "Project",
"data": [
{
"option": {
"applyDifferentContentType": true,
"allowInjection": false
},
"_id": {
"$oid": "5fddad1e48be35283c369558"
},
"url": "/project/create",
"method": "post",
"headers": {
"Content-Type": "application/json"
}
},
{
"option": {
"applyDifferentContentType": false,
"allowInjection": false
},
"_id": {
"$oid": "5fddad1e48be35283c369558"
},
"url": "/projects/update",
"method": "post",
"headers": {
"Content-Type": "application/url-encoded"
}
}
]
},
{
"_id": {
"$oid": "5fddb37d48be35283c36955e"
},
"group": "Contact",
"data": [
{
"option": {
"applyDifferentContentType": false,
"allowInjection": false
},
"_id": {
"$oid": "5fddad1e48be35283c369558"
},
"url": "/contact/create",
"method": "post",
"headers": {
"Content-Type": "multipart/form-data"
}
}
]
}
],
"__v": 0 }
Fetch Query
const result = await URLPayload.find({
"projectId": projectId,
"urls.data": {
$elemMatch: {
"_id": dataId
}
}
})
.lean();
projectId => 5fddb30d48be35283c36955a
dataId => 5fddad1e48be35283c369558
But above predicate is not giving expected result.
How to return only matched single document from nested array object ?
I have nested array object in my MongoDB document, but the predicate is not return as expected, which in this case I only want matched document only.

You can use aggregation pipeline,
make your you have to convert projectId and dataId input fields string type to object type using mongoose.Types.ObjectId
$match your conditions,
$project to show required documents
$reduce to iterate loop of urls array
$filter to iterate loop of urls.data array and filter matching _id,
$first will return object from the array, you can use $arrayElemAt instead of $first for older version of MongoDB
const result = await URLPayload.aggregate([
{
$match: {
projectId: mongoose.Types.ObjectId(projectId),
"urls.data._id": mongoose.Types.ObjectId(dataId)
}
},
{
$project: {
urlData: {
$reduce: {
input: "$urls",
initialValue: {},
in: {
$first: {
$filter: {
input: "$$this.data",
cond: { $eq: ["$$this._id", mongoose.Types.ObjectId(dataId)] }
}
}
}
}
}
}
}
])
Playground

I'm not used working with $oid inside the _id element but this should work.
const ids = {
projectId: "5fddb30d48be35283c36955a",
urlId: "5fddb34848be35283c36955c",
dataId: "5fddad1e48be35283c369558"
}
await URLPayload.findOne({ "projectId.$oid": ids.projectId },
{
"$elemMatch": { "urls.$[url].data": { "_id.$oid": ids.dataId } }
}, {
"arrayFilters": [
{ "url._id.$oid": ids.urlId }
]
}, (err, urlPayloadResult) => {
if (err) {
console.log(err)
} else {
console.log({ message: urlPayloadResult })
}
})
If not work in your case just remove the ".$oid" element.
Let me know if this work.

Related

Filter data using nodejs and elasticsearch

I'm currently facing an issue with my datatable implemented in ReactJS. I'm retrieving data from elasticsearch and populating the datatable with it. The data retrieval process works fine without the filter applied, however, when I apply filters to the data, the datatable remains empty, even though the data _source has matching records.
The structure of the parameters I am sending is as follows:
{
pageIndex: 1,
pageSize: 10,
sort: { order: '', key: '' },
query: '',
filterData: {
analysis: [ '0', '1', '2', '3' ],
threat_level_id: [ '1', '2', '3', '4' ],
}
}
EndPoint:
POST /api/v1/events/public/list
Controller:
exports.getPublicEvents = async (req, res) => {
try {
client.ping()
const { pageIndex, pageSize, sort, query, filterData } = req.body
let esQuery = {
index: 'ns_*',
body: {
query: {
bool: {
must: [
{
match_all: {},
},
],
filter: [],
},
},
from: (pageIndex - 1) * pageSize,
size: pageSize,
},
}
if (query) {
esQuery.body.query.bool.must = [
{
match: {
'Event.info': {
query: query,
fuzziness: 'AUTO',
},
},
},
]
}
if (filterData.analysis.length > 0) {
esQuery.body.query.bool.filter.push({
terms: {
'Event.analysis': filterData.analysis,
},
})
}
if (filterData.threat_level_id.length > 0) {
esQuery.body.query.bool.filter.push({
terms: {
'Event.threat_level_id': filterData.threat_level_id,
},
})
}
let esResponse = await client.search(esQuery)
let data = esResponse.hits.hits.map((hit) => hit._source)
let total = esResponse.hits.total.value
res.status(200).json({
status: 'success',
data: data,
total: total,
})
} catch (error) {
res.status(500).json({
error: 'Error connecting to Elasticsearch',
errorMessage: error.message,
})
}
}
The controller below is without filters and it works just fine.
exports.getPublicEvents = async (req, res) => {
try {
client.ping()
const { pageIndex, pageSize, sort, query } = req.body
let esQuery = {
index: 'ns_*',
body: {
query: {
match_all: {},
},
from: (pageIndex - 1) * pageSize,
size: pageSize,
},
}
if (query) {
esQuery.body.query = {
match: {
'Event.info': {
query: query,
fuzziness: 'AUTO',
},
},
}
}
let esResponse = await client.search(esQuery)
let data = esResponse.hits.hits.map((hit) => hit._source)
let total = esResponse.hits.total.value
res.status(200).json({
status: 'success',
data: data,
total: total,
})
} catch (error) {
res.status(500).json({
error: 'Error connecting to Elasticsearch',
errorMessage: error.message,
})
}
}
ElasticSearech version: 7.17.8
Result of: console.log(JSON.stringify(esQuery))
{
"index": "INDEX_NAME",
"body": {
"query": {
"bool": {
"must": [{ "match_all": {} }],
"filter": [
{ "terms": { "Event.analysis": ["0", "1", "2"] } },
{ "terms": { "Event.threat_level_id": ["1", "2", "3", "4"] } }
]
}
},
"from": 0,
"size": 10
}
}
Data in elascticsearch schema
{
"#version": "1",
"#timestamp": "2023-02-01T14:43:09.997Z",
"Event": {
"info": ".......................",
"description": ".......................",
"analysis": 0,
"threat_level_id": "4",
"created_at": 1516566351,
"uuid": "5a64f74f0e543738c12bc973322",
"updated_at": 1675262417
}
}
Index Mapping
{
"index_patterns": ["INDEX_NAME"],
"template": "TEMPLATE_NAME",
"settings": {
"number_of_replicas": 0,
"index.mapping.nested_objects.limit": 10000000
},
"mappings": {
"dynamic": false,
"properties": {
"#timestamp": {
"type": "date"
},
"Event": {
"type": "nested",
"properties": {
"date_occured": {
"type": "date"
},
"threat_level_id": {
"type": "integer"
},
"description": {
"type": "text"
},
"is_shared": {
"type": "boolean"
},
"analysis": {
"type": "integer"
},
"uuid": {
"type": "text"
},
"created_at": {
"type": "date"
},
"info": {
"type": "text"
},
"shared_with": {
"type": "nested",
"properties": {
"_id": {
"type": "text"
}
}
},
"updated_at": {
"type": "date"
},
"author": {
"type": "text"
},
"Attributes": {
"type": "nested",
"properties": {
"data": {
"type": "text"
},
"type": {
"type": "text"
},
"uuid": {
"type": "text"
},
"comment": {
"type": "text"
},
"category": {
"type": "text"
},
"value": {
"type": "text"
},
"timestamp": {
"type": "date"
}
}
},
"organisation": {
"type": "nested",
"properties": {
"name": {
"type": "text"
},
"uuid": {
"type": "text"
}
}
},
"Tags": {
"type": "nested",
"properties": {
"color": {
"type": "text"
},
"name": {
"type": "text"
}
}
},
"TLP": {
"type": "nested",
"properties": {
"color": {
"type": "text"
},
"name": {
"type": "text"
}
}
}
}
}
}
}
}
Event is a nested field, so you need to use nested queries, like this:
{
"index": "INDEX_NAME",
"body": {
"query": {
"bool": {
"must": [{ "match_all": {} }],
"filter": [
{
"nested": {
"path": "Event",
"query": {"terms": { "Event.analysis": ["0", "1", "2"] }}
}
},
{
"nested": {
"path": "Event",
"query": {"terms": { "Event.threat_level_id": ["1", "2", "3", "4"] }}
}
}
]
}
},
"from": 0,
"size": 10
}
}

How to replace a string in JSON Array using mongoose

I need to remove the domain name from the URL in the JSON array stored in Mongo DB using Mongoose.
JSON Array format in DB
{
"_id": {
"$oid": "602a482223df2a16e26f51bc"
},
"message": "Test Again",
"pollType": {
"$numberInt": "1"
},
"type": "TEST",
"optionList": [
{
"name": "name",
"original": "https://name/test/pictures/1613383695668-71393-0d1f829222f945dc",
"thumbnail": "https://name/test/pictures/1613383695668-71393-0d1f829222f945dc",
"vote": {
"$numberInt": "1"
},
"_id": {
"$oid": "602a482223df2a16e26f51b9"
},
"participateUser": [
{
"$oid": "5c9baa00bcafa608891b0b44"
}
],
"latestParticipate": [
{
"$oid": "5c9baa00bcafa608891b0b44"
}
]
},
{
"name": null,
"original": "https://name/test/pictures/1613383695672-71393-e92a34eaf6b48b19",
"thumbnail": "https://name/test/pictures/1613383695672-71393-e92a34eaf6b48b19",
"vote": {
"$numberInt": "0"
},
"_id": {
"$oid": "602a482223df2a16e26f51ba"
},
"participateUser": [
]
},
{
"name": null,
"original": "https://name/test/pictures/1613383695630-71393-3387191491ba279c",
"thumbnail": "https://name/test/pictures/1613383695630-71393-3387191491ba279c",
"vote": {
"$numberInt": "0"
},
"_id": {
"$oid": "602a482223df2a16e26f51bb"
},
"participateUser": [
]
}
],
"pollId": {
"$oid": "602a482223df2a16e26f51b8"
},
"createdAt": {
"$date": {
"$numberLong": "1613383714396"
}
},
"updatedAt": {
"$date": {
"$numberLong": "1613383714396"
}
},
"totalVoteCount": {
"$numberInt": "1"
},
"participateUser": [
{
"$oid": "5c9baa00bcafa608891b0b44"
}
]
}
I am using below code to replace a string
return new Promise(async (resolve, reject) => {
console.log("Iam called")
MongoClient.connect('mongoDBconfig', function (err, client) {
if (err) throw err;
var collection = "test"
var db = client.db('testDB');
db.collection(collection).updateMany({
"optionList.thumbnail": { $regex: /name/ },
}, {
"$set": {
"optionList.$.thumbnail": {
$replaceOne: { input: "optionList.$.thumbnail", find: "https://name/test", replacement: "" },
}
}
}
}, function (error, success) {
if (success) {
resolve(success)
}
else {
reject(error)
}
})
})
})
but when I call this function I getting below error
DB Error : Duplicate Entry : dollar ($) prefixed field '$replaceOne' in 'optionList.0.original.$replaceOne' is not valid for storage.
The $replaceOne is a aggregation operator starting from MongoDB v4.4, you can use update with aggregation pipeline starting from MongoDB v4.2
$map to iterate loop of optionList array
$replaceOne to replace string
$mergeObjects to merge current object with updated original field
let originalName = "https://name/test";
db.collection(collection).updateMany(
{ "optionList.original": { $regex: originalName } },
[{
$set: {
optionList: {
$map: {
input: "$optionList",
in: {
$mergeObjects: [
"$$this",
{
thumbnail: {
$replaceOne: {
input: "$$this.thumbnail",
find: originalName,
replacement: ""
}
}
}
]
}
}
}
}
}],
function (error, success) {
if (success) { resolve(success) }
else { reject(error) }
}
)
Playground

Removing the specific object from array in mongoose and update it

Hi i am new in mongoose and mongodb. I want to remove specific object from the Array in my document and return the updated document. I have tried a lot but it always return null. Here is my document structure.
{
"role": "Student",
"skills": [
"html",
"css",
"js"
],
"_id": "5ef583198e9b23cc8c606c10",
"user": "5ee5c9ef26333935647e54bc",
"__v": 24,
"status": "Intern",
"education": [],
"internships": [
{
"current": false,
"_id": "5ef894d48f601512340f25b5",
"title": "Web",
"company": "asdfadfd",
"from": "2010-02-04T00:00:00.000Z"
},
{
"current": false,
"_id": "5ef894f31dc9413bf89c44d8",
"title": "Django",
"company": "example",
"from": "2010-02-04T00:00:00.000Z"
}
]
}
And here is my updating function
exports.deleteStudentInternship = async (req, res, next) => {
const deleteInternship = await Student.findOneAndUpdate(
{ $and: [{ user: req.user.id }, { 'internships': { $elemMatch: { _id: req.params.intern_id } } }] },
{ '$pull': { 'internships': { _id: req.params.intern_id } } },
{
new: true,
useFindAndModify: false,
},
function (error) {
if (error) return validationError(404, { internship: 'Internship id not exist' }, next)
}
);
if (!deleteInternship) {
return validationError(404, { internship: 'Internship id not exist' }, next)
}
res.status(200).json(deleteInternship);
}
Please change the pull part I mean
{ '$pull': { 'internships': { _id: req.params.intern_id } } }
to this and try:
{ '$pull': { 'internships': req.params.intern_id } }

how to find item in collection with $match

I have collection trusted contacts. I need to filter the document by the user. When I use method findOne I have a result, but when I use $match I got an empty array. I don't know why $match doesn't work for me.
Collection trusted contacts:
{
"_id": {
"$oid": "5d76008e4b98e63e58cb34cc"
},
"date": {
"$date": "2019-09-09T07:32:20.174Z"
},
"approvedTrustedContacts": [
{
"_id": {
"$oid": "5d764e411b7476462cf6b540"
},
"user": {
"$oid": "5c5ecaf6134fc342d4b1a9d5"
}
},
{
"_id": {
"$oid": "5d7750af52352918f802c474"
},
"user": {
"$oid": "5c64968cae53a8202c963223"
}
}
],
"pendingApprovalContacts": [],
"waitingForApprovalContacts": [],
"user": {
"$oid": "5d76008e4b98e63e58cb34cb"
}
},
{
"_id": {
"$oid": "5d7605f5e7179a084efa385b"
},
"date": {
"$date": "2019-09-09T07:32:20.174Z"
},
"approvedTrustedContacts": [
{
"_id": {
"$oid": "5d764e411b7476462cf6b541"
},
"user": {
"$oid": "5d76008e4b98e63e58cb34cb"
}
}
],
"pendingApprovalContacts": [],
"waitingForApprovalContacts": [],
"user": {
"$oid": "5c5ecaf6134fc342d4b1a9d5"
}
}
when I use method findOne
const user = await TrustedContacts.findOne({ user: "5d76008e4b98e63e58cb34cb" })
I have result
but when I use $match I got empty array
result1 = await TrustedContacts.aggregate([
{ $match: { user: "5d76008e4b98e63e58cb34cb" } },
]);
It Works,
const ObjectId = require('mongodb').ObjectId;
result1 = await TrustedContacts.aggregate([
{ $match: { user: ObjectId("5d76008e4b98e63e58cb34cb") } },
]);

Is there a way with ElasticSearch to only show aggregations?

I'm trying to retrieve some data from ElasticSearch.
So far everything is working perfectly and I can query data.
But whenever I try to count a field using aggregations, the aggregation field is not in the result at the end.
So far what I've tried this as my query/function :
var client = new elasticsearch.Client({
host: 'xxxxxxxxxxxxxxxxxxxxxxx',
log:"trace"
});
client.ping({
requestTimeout: 30000,
}, function (error) {
if (error) {
console.error('elasticsearch cluster is down!');
} else {
console.log('All is well');
}
});
client.search({
"index":"worklight__appsession__1485302400000",
"type":"AppSession",
"body":{
"query": {
"filtered": {
"query": {
"query_string": {
"analyze_wildcard": true,
"query": "*"
}
},
"filter": {
"bool": {
"must": [
{
"range": {
"timestamp": {
"gte": 1553507131976,
"lte": 1553593531976
}
}
}
],
"must_not": []
}
}
}
},
"aggs": {
"1": {
"cardinality": {
"field": "deviceID"
}
}
}
}
}).then(function (resp) {
var hits = resp.hits.hits;
console.log(hits)
}, function (err) {
console.trace(err.message);
});
and the result is :
Elasticsearch DEBUG: 2019-03-26T09:46:21Z
starting request {
"method": "HEAD",
"requestTimeout": 30000,
"castExists": true,
"path": "/",
"query": {}
}
Elasticsearch DEBUG: 2019-03-26T09:46:21Z
starting request {
"method": "POST",
"path": "/worklight__appsession__1485302400000/AppSession/_search",
"body": {
"query": {
"filtered": {
"query": {
"query_string": {
"analyze_wildcard": true,
"query": "*"
}
},
"filter": {
"bool": {
"must": [
{
"range": {
"timestamp": {
"gte": 1553507131976,
"lte": 1553593531976
}
}
}
],
"must_not": []
}
}
}
},
"aggs": {
"1": {
"cardinality": {
"field": "deviceID"
}
}
}
},
"query": {}
}
Elasticsearch TRACE: 2019-03-26T09:46:22Z
-> HEAD http://xx/
<- 200
Elasticsearch DEBUG: 2019-03-26T09:46:22Z
Request complete
All is well
Elasticsearch TRACE: 2019-03-26T09:46:22Z
-> POST http://xx/worklight__appsession__1485302400000/AppSession/_search
{
"query": {
"filtered": {
"query": {
"query_string": {
"analyze_wildcard": true,
"query": "*"
}
},
"filter": {
"bool": {
"must": [
{
"range": {
"timestamp": {
"gte": 1553507131976,
"lte": 1553593531976
}
}
}
],
"must_not": []
}
}
}
},
"aggs": {
"1": {
"cardinality": {
"field": "deviceID"
}
}
}
}
<- 200
{
"took": 4,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"failed": 0
},
"hits": {
"total": 325,
"max_score": 1,
"hits": [
...
confidential data here, not relevant to the topic.
...
}
]
},
"aggregations": {
"1": {
"value": 133
}
}
}
But if erase the log trace option, aggregations don't show up in the result :
[ { _index: 'worklight__appsession__1485302400000',
_type: 'AppSession',
... Some Data,
{ _index: 'worklight__appsession__1485302400000',
_type: 'AppSession',
... Some Data,
{ _index: 'worklight__appsession__1485302400000',
_type: 'AppSession',
... Some Data,
]
Am I doing something wrong, or do I just lack knowledge ?
Thanks for your time.
You are doing console.log(resp.hits.hits). Try this instead:
console.log(resp.aggregations)

Resources