findByIdAndDelete not finding the provided id - node.js

I am trying to use Mongoose findByIdAndDelete() method on a model but it does not see the _id it needs to delete.
My route is declared similar to this (minified) example:
router.delete('/delete-entity/:id', (req, res) => {
Entity.findByIdAndDelete(req.params.id)
.exec()
.then((docs) => { ...handle success })
.catch((err) => { ...handle error });
});
I only have middleware running that hooks into the save method of the model. But this middleware does not touch the _id field.
When I console.log() inside the route I can see that req.params.id is set. Also if I try to identify my document using an other field to delete it (so not the _id field), it does works. However when I try to use the _id field all that is returned to the client is ''. This makes me believe that something goes wrong in comparing datatypes of the _id field. But cating using mongoose.Type.ObjectId() did not help either.
What could be going on here?
In addition to this already provided information. Here is my model.
const embeddedEntity = new mongoose.Schema({
name: { type: String, required: true }
}, { _id: false });
const entity = new mongoose.Schema({
name: { type: String, required: true },
embeddedInfo: { type: embeddedEntity, required: true }
});
module.exports = mongoose.model('Entity', entity);
#Invider I managed to get this from the profiler.
{
"op" : "command",
"ns" : "development.entities",
"command" : {
"findAndModify" : "entities",
"query" : {
"_id" : ObjectId("5dc95b8cc472d31232dba5a3")
},
"new" : false,
"remove" : true,
"upsert" : false,
"lsid" : {
"id" : UUID("30086660-0619-440e-9268-148957428a2b")
},
"$db" : "development"
},
"keysExamined" : 0,
"docsExamined" : 0,
"ndeleted" : 0,
"numYield" : 0,
"locks" : {
"Global" : {
"acquireCount" : {
"r" : NumberLong(1),
"w" : NumberLong(1)
}
},
"Database" : {
"acquireCount" : {
"w" : NumberLong(1)
}
},
"Collection" : {
"acquireCount" : {
"w" : NumberLong(1)
}
}
},
"responseLength" : 74,
"protocol" : "op_msg",
"millis" : 0,
"planSummary" : "IDHACK",
"execStats" : {
"stage" : "DELETE",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"nWouldDelete" : 0,
"nInvalidateSkips" : 0,
"inputStage" : {
"stage" : "IDHACK",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"keysExamined" : 0,
"docsExamined" : 0
}
},
"ts" : ISODate("2019-11-14T09:27:52.988Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}

A lot of thanks goes out to #invider and #SuleymanSah for pointing me into the right direction.
The problem was actually something pretty simple that I completely overlooked and was indeed related to the datatype as I expected.
My e2e tests were failing because of the same problem as trying to call the route using Postman.
The problem was that when adding test samples to the database I was setting the _id field myself. This most likely causes the value stored to be of type string. Letting MongoDB itself set the _id field solved this issue. At least for working in the live environment using Postman.
For my e2e test I had to modify my the data that will be loaded into the database for each test. I was adding samples in the following way:
{
_id: '5dc95b8cc472d31232dba5a5',
name: 'SomeData',
embeddedInfo: {
name: 'someOtherData'
}
}
I should have casted the _id into an ObjectId first. Like so:
{
_id: mongoose.Types.ObjectId('5dc95b8cc472d31232dba5a5'),
name: 'SomeData',
embeddedInfo: {
name: 'someOtherData'
}
}
Again. Thanks to the both of you.

Related

High load on simple Mongo find with indices

I have a mongoDB I'm querying using NodeJS (running mongoose).
In this particular case I'm querying a bunch of collections and pipe the data as CSV into archiverjs to create a zip file. So I have an incoming request, the data gets queried using mongoose and a mongo cursor, piped into a pipeline which will end in archiverjs respectively the http response delivering the zip file to the user.
async function getSortedQueryCursor(...) {
...
const query = MODEL_LOOKUP[fileType]
.find(reducer)
.sort({ [idString]: 'asc' });
return query.cursor();
}
async function getData(...) {
const cursor = await getSortedQueryCursor(...);
return cursor
.pipe(filter1Stream)
.pipe(filter2Stream)
.pipe(filter3Stream)
.pipe(csvStringifyStream);
}
router.post('/:scenarioId', async (request, response) => {
...
const archive = Archiver(...);
archive.pipe(response);
const result = await getData(...);
archive.append(stream, { name: filepath });
return archive.finalize();
}
As soon as a particular collection is in the game (the collection holds roughly 40 million documents) the query lasts very long (>15s) and I can see the mongo process on 100% CPU during that time. Even more surprising as the result set is empty (no documents matching the query).
It's a rather simple query:
items.find({ scenarioId: 'ckqf5ulg38gu208eecxlf95fc' }, { sort: { dataId: 1 }
I have indices on scenarioId and dataId. If I run the query on the shell it returns in 30ms.
An explain() results in:
[
{
"queryPlanner": {
"plannerVersion": 1,
"namespace": "data.items",
"indexFilterSet": false,
"parsedQuery": {
"scenarioId": {
"$eq": "ckqf5ulg38gu208eecxlf95fc"
}
},
"winningPlan": {
"stage": "SORT",
"sortPattern": {
"itemId": 1
},
"memLimit": 104857600,
"type": "simple",
"inputStage": {
"stage": "FETCH",
"inputStage": {
"stage": "IXSCAN",
"keyPattern": {
"scenarioId": 1
},
"indexName": "scenarioId_1",
"isMultiKey": false,
"multiKeyPaths": {
"scenarioId": []
},
"isUnique": false,
"isSparse": false,
"isPartial": false,
"indexVersion": 2,
"direction": "forward",
"indexBounds": {
"scenarioId": [
"[\"ckqf5ulg38gu208eecxlf95fc\", \"ckqf5ulg38gu208eecxlf95fc\"]"
]
}
}
}
},
"rejectedPlans": [
...
]
},
"executionStats": {
"executionSuccess": true,
"nReturned": 0,
"executionTimeMillis": 0,
"totalKeysExamined": 0,
"totalDocsExamined": 0,
"executionStages": {
"stage": "SORT",
"nReturned": 0,
"executionTimeMillisEstimate": 0,
"works": 3,
"advanced": 0,
"needTime": 1,
"needYield": 0,
"saveState": 0,
"restoreState": 0,
"isEOF": 1,
"sortPattern": {
"dataId": 1
},
"memLimit": 104857600,
"type": "simple",
"totalDataSizeSorted": 0,
"usedDisk": false,
"inputStage": {
"stage": "FETCH",
"nReturned": 0,
"executionTimeMillisEstimate": 0,
"works": 1,
"advanced": 0,
"needTime": 0,
"needYield": 0,
"saveState": 0,
"restoreState": 0,
"isEOF": 1,
"docsExamined": 0,
"alreadyHasObj": 0,
"inputStage": {
"stage": "IXSCAN",
"nReturned": 0,
"executionTimeMillisEstimate": 0,
"works": 1,
"advanced": 0,
"needTime": 0,
"needYield": 0,
"saveState": 0,
"restoreState": 0,
"isEOF": 1,
"keyPattern": {
"scenarioId": 1
},
"indexName": "scenarioId_1",
"isMultiKey": false,
"multiKeyPaths": {
"scenarioId": []
},
"isUnique": false,
"isSparse": false,
"isPartial": false,
"indexVersion": 2,
"direction": "forward",
"indexBounds": {
"scenarioId": [
"[\"ckqf5ulg38gu208eecxlf95fc\", \"ckqf5ulg38gu208eecxlf95fc\"]"
]
},
"keysExamined": 0,
"seeks": 1,
"dupsTested": 0,
"dupsDropped": 0
}
}
},
...
},
"serverInfo": {
...
"version": "4.4.6",
"gitVersion": "72e66213c2c3eab37d9358d5e78ad7f5c1d0d0d7"
},
...
}
]
It tells me (I'm not very experienced in interpreting those results) that the query is quite cheap: "executionTimeMillisEstimate": 0, as it's not running a document scan "docsExamined": 0,.
Next I connected to the mongo server and ran db.currentOp({"secs_running": {$gte: 5}}) to get some information from this side:
{
"type" : "op",
...
"clientMetadata" : {
"driver" : {
"name" : "nodejs|Mongoose",
"version" : "3.6.5"
},
"os" : {
"type" : "Linux",
"name" : "linux",
"architecture" : "x64",
"version" : "5.8.0-50-generic"
},
"platform" : "'Node.js v14.17.0, LE (unified)",
"version" : "3.6.5|5.12.3"
},
"active" : true,
"secs_running" : NumberLong(16),
"microsecs_running" : NumberLong(16661409),
"op" : "query",
"ns" : "data.items",
"command" : {
"find" : "items",
"filter" : {
"scenarioId" : "ckqf5ulg38gu208eecxlf95fc"
},
"sort" : {
"itemId" : 1
},
"projection" : {
},
"returnKey" : false,
"showRecordId" : false,
"lsid" : {
"id" : UUID("be3ce18b-5365-4680-b734-543d06418301")
},
"$clusterTime" : {
"clusterTime" : Timestamp(1625498044, 1),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : 0
}
},
"$db" : "data",
"$readPreference" : {
"mode" : "primaryPreferred"
}
},
"numYields" : 14701,
"locks" : {
"ReplicationStateTransition" : "w",
"Global" : "r",
"Database" : "r",
"Collection" : "r"
},
"waitingForLock" : false,
"lockStats" : {
"ReplicationStateTransition" : {
"acquireCount" : {
"w" : NumberLong(14702)
}
},
"Global" : {
"acquireCount" : {
"r" : NumberLong(14702)
}
},
"Database" : {
"acquireCount" : {
"r" : NumberLong(14702)
}
},
"Collection" : {
"acquireCount" : {
"r" : NumberLong(14702)
}
},
"Mutex" : {
"acquireCount" : {
"r" : NumberLong(1)
}
}
},
"waitingForFlowControl" : false,
"flowControlStats" : {
}
}
Any ideas how to improve the performance or find the bottleneck in my application? As the load is high on mongo side and no documents are found/passed to the application I guess it's mongo having trouble ...
EDIT: I've logged the whole process from DB side using db.setProfilingLevel(2) and db.system.profile.find().pretty(). Here we can see that the whole collection (or am I misinterpreting "docsExamined" : 39612167?) is queried:
{
"op" : "query",
"ns" : "data.items",
"command" : {
"find" : "items",
"filter" : {
"scenarioId" : "ckqf5ulg38gu208eecxlf95fc"
},
"sort" : {
"dataId" : 1
},
"projection" : {
},
...
"$db" : "data",
"$readPreference" : {
"mode" : "primaryPreferred"
}
},
"keysExamined" : 39612167,
"docsExamined" : 39612167,
"cursorExhausted" : true,
"numYield" : 39613,
"nreturned" : 0,
"queryHash" : "B7F40289",
"planCacheKey" : "BADED068",
"locks" : {
"ReplicationStateTransition" : {
"acquireCount" : {
"w" : NumberLong(39615)
}
},
"Global" : {
"acquireCount" : {
"r" : NumberLong(39615)
}
},
"Database" : {
"acquireCount" : {
"r" : NumberLong(39614)
}
},
"Collection" : {
"acquireCount" : {
"r" : NumberLong(39614)
}
},
"Mutex" : {
"acquireCount" : {
"r" : NumberLong(1)
}
}
},
"flowControl" : {
},
"storage" : {
},
"responseLength" : 242,
"protocol" : "op_msg",
"millis" : 48401,
"planSummary" : "IXSCAN { dataId: 1 }",
"execStats" : {
"stage" : "CACHED_PLAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 48401,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 39613,
"restoreState" : 39613,
"isEOF" : 1,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"scenarioId" : {
"$eq" : "ckqf5ulg38gu208eecxlf95fc"
}
},
"nReturned" : 0,
"executionTimeMillisEstimate" : 6270,
"works" : 39612168,
"advanced" : 0,
"needTime" : 39612167,
"needYield" : 0,
"saveState" : 39613,
"restoreState" : 39613,
"isEOF" : 1,
"docsExamined" : 39612167,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 39612167,
"executionTimeMillisEstimate" : 2151,
"works" : 39612168,
"advanced" : 39612167,
"needTime" : 0,
"needYield" : 0,
"saveState" : 39613,
"restoreState" : 39613,
"isEOF" : 1,
"keyPattern" : {
"dataId" : 1
},
"indexName" : "dataId_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"dataId" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"dataId" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 39612167,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
(As usual) it seems like the indices are not properly set. I've created a new (secondary?) index:
{
"dataId" : 1,
"scenarioId": 1
}
and now the query returns within milliseconds ...
EDIT: What still makes me wonder is that the shell query returned in milliseconds and the mongoose query took ages. Even though the queries seem to be identical (from my point of view) mongo treats them differently.

mongodb / nodejs: findandmodify method not working

I have below json data and want to update the value according to the condition
{
"_id" : ObjectId("5fce2d4c7b2ea2e79ercvju4"),
"userId" : 1,
"token": "jwt_token_value",
"isActive" : 0,
"__v" : 0
}
{
"_id" : ObjectId("5fce2d4c7b2ea2e79ercvjk0"),
"userId" : 1,
"token": "jwt_token_value",
"isActive" : 0,
"__v" : 0
}
{
"_id" : ObjectId("5fd0d45tjd82a02dd0f17fc4"),
"userId" : 1,
"token": "jwt_token_value",
"isActive" : 1,
"__v" : 0
}
I have managed the things as below.
let update = await UserDetails.findAndModify(
{ userId: 1, token: token },
[],
{$set: { isActive: 0 }},
{ new: true }
);
=> This query should update the last json collection key of isActive to 1. But it is not updating the values any how. What the things that I am doing wrong ? It is not throwing me any error as well to debug.
I am following this answer: https://stackoverflow.com/a/24648693/5624578
You can include the query identifiers to specify what each section is doing like this:
let update = await UserDetails.findAndModify(
query: { userId: 1, token: token },
update: {$set: { isActive: 0 }},
upsert: true
);

How can I update a subdocument in MongoDB?

Code created in Mongoose to update a subdocument was not working. So I tried to update the subdocument within the Mongo Shell.
This is the document (location) and subdocument (review):
{
"_id" : ObjectId("56d8c73314fbc7e702cfb8c4"),
"name" : "Costly",
"address" : "150, Super Street",
"coords" : [
-0.9630884,
51.451041
],
"reviews" : [
{
"author" : "kyle riggen1",
"_id" : ObjectId("56d8de74cc7f953efd8455d9"),
"rating" : 4,
"timestamp" : ISODate("2015-06-01T06:00:00Z"),
"reviewText" : "will the ID work?"
}
],
"rating" : 0,
"__v" : 2
}
Here are some of my attempts at updating the subdocument:
This question gave this format:
update({
_id: "56d8c73314fbc7e702cfb8c4",
"reviews._id": ObjectId("56d8de74cc7f953efd8455d9")
},{
$set: {"reviews.$.rating": 1}
}, false, true
);
This returned an error of "update is not defined" as shown:
2016-03-03T22:52:44.445-0700 E QUERY [thread1] ReferenceError: update is not defined :
#(shell):1:1
Which I think is because the command did not start with db.locations.update()
MongoDB documentation used this format:
db.locations.update(
{
_id: "56d8c73314fbc7e702cfb8c4",
review: { $elemMatch: { author: "kyle riggen1" } }
},
{ $set: { "location.$.rating" : 1 } }
)
This returns a valid update but the update didn't actually happen as shown:
WriteResult({ "nMatched" : 0, "nUpserted" : 0, "nModified" : 0 })
This question used this format:
db.locations.update({
_id: "56d8c73314fbc7e702cfb8c4",
'review.author': 'kyle riggen1'
},
{ $set: { 'review.$.rating': 1 }}
)
This returns the same as the MongoDB documentation as shown here:
WriteResult({ "nMatched" : 0, "nUpserted" : 0, "nModified" : 0 })
So these queries I guess are working but my data is not getting updated. Would my data be indexed wrong perhaps? The actual location is able to be updated even through a Mongoose API.
You can do it By $push Or $addToSet.
db.col.update(
{ name: 'reviews', 'list.id': 2 },
{$push: {'list.$.items': {id: 5, name: 'item5'}}}
)
See the reference from mongodb Manual
https://docs.mongodb.org/manual/reference/operator/update/push/
Please know db.collection.update( criteria, objNew, upsert, multi )
criteria: match condition
objNew: update content
upsert: true or false
true : if not existed, insert it
false : if not existed, don't insert
multi: true or false
true : update all matched documents
false : only update the first matched document

How to add data in nested array in mongoshell

I have a schema :
{
"id": String,
"username": String,
"password": String,
"email": String,
"firstName": String,
"lastName": String,
"system" : {
"item" : {type: Number},
"update_interval" : { type: Number, max: 99999 },
"reading" : [
{
"id" : { type: Number},
"adc1" : { type: Number, max: 4095 },
"adc2" : { type: Number, max: 4095 },
"pc_datestamp" :Date,
}
]
}
now i want to add values to
"reading" : [
{
"id" : { type: Number},
"adc1" : { type: Number, max: 4095 },
"adc2" : { type: Number, max: 4095 },
"pc_datestamp" :Date,
}
]
but i dont know where I am wrong I have tried to update data from mongoshell but no success till now
> db.users.update( {"email" : "test#test.com", "system.item": 1, }, {"$push": {"system.$.reading": [{"adc1" : "123", "adc2": "1245", "id":"1" }] } })
WriteResult({
"nMatched" : 0,
"nUpserted" : 0,
"nModified" : 0,
"writeError" : {
"code" : 16837,
"errmsg" : "The positional operator did not find the match needed from the query. Unexpanded update: system.$.reading"
}
> db.users.update( {"email" : "test#test.com", "system": {$elemMatch:{ item: 1}} }, {"$push": {"system.$.reading": {"adc1" : "123", "adc2": "1245", "id":"1" } } })
WriteResult({ "nMatched" : 0, "nUpserted" : 0, "nModified" : 0 })
I have set the value of item as one
> db.users.find( {"email" : "test#test.com", "system.item": 1} ).pretty()
{
"_id" : ObjectId("56dd88578ff7fbd714091a4a"),
"lastName" : "test",
"firstName" : "test",
"email" : "test#test.com",
"password" : "$2a$10$wY9wr9oreza4fBX7CfXym.rPZUPrcesigYIfWd0zbM4dDjBy6k3vy",
"username" : "test",
"system" : {
"item" : 1,
"reading" : [ ]
},
"__v" : 0
}
I have followed
Mongodb $push in nested array
and this
Insert data in nested array in mongodb
and many more questions but cannot find whats wrong.
Since the positional $ operator acts as a placeholder for the first element that matches the query document, and the array field must appear as part of the query document, your update
operation does not satisfy these conditions hence it suffers from the error fate you are getting. In your query you are only referencing the "system.item" field which is not an array.
The other option you can do is ditch the positional $ operator in your update and just add the object to the array using $addToset which adds elements to an array only if they do not already exist in the set:
db.users.update(
{
"email": "test#test.com",
"system.item": 1
},
{
"$addToSet": {
"system.reading": {
"adc1" : "123",
"adc2": "1245",
"id":"1"
}
}
}
)

mongoose update data information, how to prevent overwrite

So i was trying to update data store in the mongodb database, using Mongoose
original data structure looks like this
{
"_id" : ObjectId("234u232kjrkjwebrkw"),
"local" : {
"password" : "sdflsdjflsdjlfkjsdlkfjklsdjflksd",
"email" : "email#email.com"
},
"__v" : 0
}
I'm trying to update the "userName" property in the "local". after which it supposed to be
{
"_id" : ObjectId("234u232kjrkjwebrkw"),
"local" : {
"password" : "sdflsdjflsdjlfkjsdlkfjklsdjflksd",
"email" : "email#email.com",
"userName" : "yowhatsup"
},
"__v" : 0
}
I used this
User.findByIdAndUpdate("54a490ab6e13cca1d47870d6", {local:{ userName: 'jasonBorne' }}, { upsert: true }, function(){})
it became this
{
"_id" : ObjectId("234u232kjrkjwebrkw"),
"local" : {
"userName" : "yowhatsup"
},
"__v" : 0
}
it's been overwritten.
how to avoid this?
note: the new "userName" is defined as {type:String,default:null} when the model was structred.
You can do an an update if exists, otherwise insert:
Arguments for update are: findQuery, data, queryOptions, onComplete
User.update({"_id" : ObjectId("234u232kjrkjwebrkw")}, { $set: { "local.userName": "whatsup" } }, { upsert: true }, function(err){...});

Resources