I have a mongoDB I'm querying using NodeJS (running mongoose).
In this particular case I'm querying a bunch of collections and pipe the data as CSV into archiverjs to create a zip file. So I have an incoming request, the data gets queried using mongoose and a mongo cursor, piped into a pipeline which will end in archiverjs respectively the http response delivering the zip file to the user.
async function getSortedQueryCursor(...) {
...
const query = MODEL_LOOKUP[fileType]
.find(reducer)
.sort({ [idString]: 'asc' });
return query.cursor();
}
async function getData(...) {
const cursor = await getSortedQueryCursor(...);
return cursor
.pipe(filter1Stream)
.pipe(filter2Stream)
.pipe(filter3Stream)
.pipe(csvStringifyStream);
}
router.post('/:scenarioId', async (request, response) => {
...
const archive = Archiver(...);
archive.pipe(response);
const result = await getData(...);
archive.append(stream, { name: filepath });
return archive.finalize();
}
As soon as a particular collection is in the game (the collection holds roughly 40 million documents) the query lasts very long (>15s) and I can see the mongo process on 100% CPU during that time. Even more surprising as the result set is empty (no documents matching the query).
It's a rather simple query:
items.find({ scenarioId: 'ckqf5ulg38gu208eecxlf95fc' }, { sort: { dataId: 1 }
I have indices on scenarioId and dataId. If I run the query on the shell it returns in 30ms.
An explain() results in:
[
{
"queryPlanner": {
"plannerVersion": 1,
"namespace": "data.items",
"indexFilterSet": false,
"parsedQuery": {
"scenarioId": {
"$eq": "ckqf5ulg38gu208eecxlf95fc"
}
},
"winningPlan": {
"stage": "SORT",
"sortPattern": {
"itemId": 1
},
"memLimit": 104857600,
"type": "simple",
"inputStage": {
"stage": "FETCH",
"inputStage": {
"stage": "IXSCAN",
"keyPattern": {
"scenarioId": 1
},
"indexName": "scenarioId_1",
"isMultiKey": false,
"multiKeyPaths": {
"scenarioId": []
},
"isUnique": false,
"isSparse": false,
"isPartial": false,
"indexVersion": 2,
"direction": "forward",
"indexBounds": {
"scenarioId": [
"[\"ckqf5ulg38gu208eecxlf95fc\", \"ckqf5ulg38gu208eecxlf95fc\"]"
]
}
}
}
},
"rejectedPlans": [
...
]
},
"executionStats": {
"executionSuccess": true,
"nReturned": 0,
"executionTimeMillis": 0,
"totalKeysExamined": 0,
"totalDocsExamined": 0,
"executionStages": {
"stage": "SORT",
"nReturned": 0,
"executionTimeMillisEstimate": 0,
"works": 3,
"advanced": 0,
"needTime": 1,
"needYield": 0,
"saveState": 0,
"restoreState": 0,
"isEOF": 1,
"sortPattern": {
"dataId": 1
},
"memLimit": 104857600,
"type": "simple",
"totalDataSizeSorted": 0,
"usedDisk": false,
"inputStage": {
"stage": "FETCH",
"nReturned": 0,
"executionTimeMillisEstimate": 0,
"works": 1,
"advanced": 0,
"needTime": 0,
"needYield": 0,
"saveState": 0,
"restoreState": 0,
"isEOF": 1,
"docsExamined": 0,
"alreadyHasObj": 0,
"inputStage": {
"stage": "IXSCAN",
"nReturned": 0,
"executionTimeMillisEstimate": 0,
"works": 1,
"advanced": 0,
"needTime": 0,
"needYield": 0,
"saveState": 0,
"restoreState": 0,
"isEOF": 1,
"keyPattern": {
"scenarioId": 1
},
"indexName": "scenarioId_1",
"isMultiKey": false,
"multiKeyPaths": {
"scenarioId": []
},
"isUnique": false,
"isSparse": false,
"isPartial": false,
"indexVersion": 2,
"direction": "forward",
"indexBounds": {
"scenarioId": [
"[\"ckqf5ulg38gu208eecxlf95fc\", \"ckqf5ulg38gu208eecxlf95fc\"]"
]
},
"keysExamined": 0,
"seeks": 1,
"dupsTested": 0,
"dupsDropped": 0
}
}
},
...
},
"serverInfo": {
...
"version": "4.4.6",
"gitVersion": "72e66213c2c3eab37d9358d5e78ad7f5c1d0d0d7"
},
...
}
]
It tells me (I'm not very experienced in interpreting those results) that the query is quite cheap: "executionTimeMillisEstimate": 0, as it's not running a document scan "docsExamined": 0,.
Next I connected to the mongo server and ran db.currentOp({"secs_running": {$gte: 5}}) to get some information from this side:
{
"type" : "op",
...
"clientMetadata" : {
"driver" : {
"name" : "nodejs|Mongoose",
"version" : "3.6.5"
},
"os" : {
"type" : "Linux",
"name" : "linux",
"architecture" : "x64",
"version" : "5.8.0-50-generic"
},
"platform" : "'Node.js v14.17.0, LE (unified)",
"version" : "3.6.5|5.12.3"
},
"active" : true,
"secs_running" : NumberLong(16),
"microsecs_running" : NumberLong(16661409),
"op" : "query",
"ns" : "data.items",
"command" : {
"find" : "items",
"filter" : {
"scenarioId" : "ckqf5ulg38gu208eecxlf95fc"
},
"sort" : {
"itemId" : 1
},
"projection" : {
},
"returnKey" : false,
"showRecordId" : false,
"lsid" : {
"id" : UUID("be3ce18b-5365-4680-b734-543d06418301")
},
"$clusterTime" : {
"clusterTime" : Timestamp(1625498044, 1),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : 0
}
},
"$db" : "data",
"$readPreference" : {
"mode" : "primaryPreferred"
}
},
"numYields" : 14701,
"locks" : {
"ReplicationStateTransition" : "w",
"Global" : "r",
"Database" : "r",
"Collection" : "r"
},
"waitingForLock" : false,
"lockStats" : {
"ReplicationStateTransition" : {
"acquireCount" : {
"w" : NumberLong(14702)
}
},
"Global" : {
"acquireCount" : {
"r" : NumberLong(14702)
}
},
"Database" : {
"acquireCount" : {
"r" : NumberLong(14702)
}
},
"Collection" : {
"acquireCount" : {
"r" : NumberLong(14702)
}
},
"Mutex" : {
"acquireCount" : {
"r" : NumberLong(1)
}
}
},
"waitingForFlowControl" : false,
"flowControlStats" : {
}
}
Any ideas how to improve the performance or find the bottleneck in my application? As the load is high on mongo side and no documents are found/passed to the application I guess it's mongo having trouble ...
EDIT: I've logged the whole process from DB side using db.setProfilingLevel(2) and db.system.profile.find().pretty(). Here we can see that the whole collection (or am I misinterpreting "docsExamined" : 39612167?) is queried:
{
"op" : "query",
"ns" : "data.items",
"command" : {
"find" : "items",
"filter" : {
"scenarioId" : "ckqf5ulg38gu208eecxlf95fc"
},
"sort" : {
"dataId" : 1
},
"projection" : {
},
...
"$db" : "data",
"$readPreference" : {
"mode" : "primaryPreferred"
}
},
"keysExamined" : 39612167,
"docsExamined" : 39612167,
"cursorExhausted" : true,
"numYield" : 39613,
"nreturned" : 0,
"queryHash" : "B7F40289",
"planCacheKey" : "BADED068",
"locks" : {
"ReplicationStateTransition" : {
"acquireCount" : {
"w" : NumberLong(39615)
}
},
"Global" : {
"acquireCount" : {
"r" : NumberLong(39615)
}
},
"Database" : {
"acquireCount" : {
"r" : NumberLong(39614)
}
},
"Collection" : {
"acquireCount" : {
"r" : NumberLong(39614)
}
},
"Mutex" : {
"acquireCount" : {
"r" : NumberLong(1)
}
}
},
"flowControl" : {
},
"storage" : {
},
"responseLength" : 242,
"protocol" : "op_msg",
"millis" : 48401,
"planSummary" : "IXSCAN { dataId: 1 }",
"execStats" : {
"stage" : "CACHED_PLAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 48401,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 39613,
"restoreState" : 39613,
"isEOF" : 1,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"scenarioId" : {
"$eq" : "ckqf5ulg38gu208eecxlf95fc"
}
},
"nReturned" : 0,
"executionTimeMillisEstimate" : 6270,
"works" : 39612168,
"advanced" : 0,
"needTime" : 39612167,
"needYield" : 0,
"saveState" : 39613,
"restoreState" : 39613,
"isEOF" : 1,
"docsExamined" : 39612167,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 39612167,
"executionTimeMillisEstimate" : 2151,
"works" : 39612168,
"advanced" : 39612167,
"needTime" : 0,
"needYield" : 0,
"saveState" : 39613,
"restoreState" : 39613,
"isEOF" : 1,
"keyPattern" : {
"dataId" : 1
},
"indexName" : "dataId_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"dataId" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"dataId" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 39612167,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
(As usual) it seems like the indices are not properly set. I've created a new (secondary?) index:
{
"dataId" : 1,
"scenarioId": 1
}
and now the query returns within milliseconds ...
EDIT: What still makes me wonder is that the shell query returned in milliseconds and the mongoose query took ages. Even though the queries seem to be identical (from my point of view) mongo treats them differently.
I have below json data and want to update the value according to the condition
{
"_id" : ObjectId("5fce2d4c7b2ea2e79ercvju4"),
"userId" : 1,
"token": "jwt_token_value",
"isActive" : 0,
"__v" : 0
}
{
"_id" : ObjectId("5fce2d4c7b2ea2e79ercvjk0"),
"userId" : 1,
"token": "jwt_token_value",
"isActive" : 0,
"__v" : 0
}
{
"_id" : ObjectId("5fd0d45tjd82a02dd0f17fc4"),
"userId" : 1,
"token": "jwt_token_value",
"isActive" : 1,
"__v" : 0
}
I have managed the things as below.
let update = await UserDetails.findAndModify(
{ userId: 1, token: token },
[],
{$set: { isActive: 0 }},
{ new: true }
);
=> This query should update the last json collection key of isActive to 1. But it is not updating the values any how. What the things that I am doing wrong ? It is not throwing me any error as well to debug.
I am following this answer: https://stackoverflow.com/a/24648693/5624578
You can include the query identifiers to specify what each section is doing like this:
let update = await UserDetails.findAndModify(
query: { userId: 1, token: token },
update: {$set: { isActive: 0 }},
upsert: true
);
Code created in Mongoose to update a subdocument was not working. So I tried to update the subdocument within the Mongo Shell.
This is the document (location) and subdocument (review):
{
"_id" : ObjectId("56d8c73314fbc7e702cfb8c4"),
"name" : "Costly",
"address" : "150, Super Street",
"coords" : [
-0.9630884,
51.451041
],
"reviews" : [
{
"author" : "kyle riggen1",
"_id" : ObjectId("56d8de74cc7f953efd8455d9"),
"rating" : 4,
"timestamp" : ISODate("2015-06-01T06:00:00Z"),
"reviewText" : "will the ID work?"
}
],
"rating" : 0,
"__v" : 2
}
Here are some of my attempts at updating the subdocument:
This question gave this format:
update({
_id: "56d8c73314fbc7e702cfb8c4",
"reviews._id": ObjectId("56d8de74cc7f953efd8455d9")
},{
$set: {"reviews.$.rating": 1}
}, false, true
);
This returned an error of "update is not defined" as shown:
2016-03-03T22:52:44.445-0700 E QUERY [thread1] ReferenceError: update is not defined :
#(shell):1:1
Which I think is because the command did not start with db.locations.update()
MongoDB documentation used this format:
db.locations.update(
{
_id: "56d8c73314fbc7e702cfb8c4",
review: { $elemMatch: { author: "kyle riggen1" } }
},
{ $set: { "location.$.rating" : 1 } }
)
This returns a valid update but the update didn't actually happen as shown:
WriteResult({ "nMatched" : 0, "nUpserted" : 0, "nModified" : 0 })
This question used this format:
db.locations.update({
_id: "56d8c73314fbc7e702cfb8c4",
'review.author': 'kyle riggen1'
},
{ $set: { 'review.$.rating': 1 }}
)
This returns the same as the MongoDB documentation as shown here:
WriteResult({ "nMatched" : 0, "nUpserted" : 0, "nModified" : 0 })
So these queries I guess are working but my data is not getting updated. Would my data be indexed wrong perhaps? The actual location is able to be updated even through a Mongoose API.
You can do it By $push Or $addToSet.
db.col.update(
{ name: 'reviews', 'list.id': 2 },
{$push: {'list.$.items': {id: 5, name: 'item5'}}}
)
See the reference from mongodb Manual
https://docs.mongodb.org/manual/reference/operator/update/push/
Please know db.collection.update( criteria, objNew, upsert, multi )
criteria: match condition
objNew: update content
upsert: true or false
true : if not existed, insert it
false : if not existed, don't insert
multi: true or false
true : update all matched documents
false : only update the first matched document
I have a schema :
{
"id": String,
"username": String,
"password": String,
"email": String,
"firstName": String,
"lastName": String,
"system" : {
"item" : {type: Number},
"update_interval" : { type: Number, max: 99999 },
"reading" : [
{
"id" : { type: Number},
"adc1" : { type: Number, max: 4095 },
"adc2" : { type: Number, max: 4095 },
"pc_datestamp" :Date,
}
]
}
now i want to add values to
"reading" : [
{
"id" : { type: Number},
"adc1" : { type: Number, max: 4095 },
"adc2" : { type: Number, max: 4095 },
"pc_datestamp" :Date,
}
]
but i dont know where I am wrong I have tried to update data from mongoshell but no success till now
> db.users.update( {"email" : "test#test.com", "system.item": 1, }, {"$push": {"system.$.reading": [{"adc1" : "123", "adc2": "1245", "id":"1" }] } })
WriteResult({
"nMatched" : 0,
"nUpserted" : 0,
"nModified" : 0,
"writeError" : {
"code" : 16837,
"errmsg" : "The positional operator did not find the match needed from the query. Unexpanded update: system.$.reading"
}
> db.users.update( {"email" : "test#test.com", "system": {$elemMatch:{ item: 1}} }, {"$push": {"system.$.reading": {"adc1" : "123", "adc2": "1245", "id":"1" } } })
WriteResult({ "nMatched" : 0, "nUpserted" : 0, "nModified" : 0 })
I have set the value of item as one
> db.users.find( {"email" : "test#test.com", "system.item": 1} ).pretty()
{
"_id" : ObjectId("56dd88578ff7fbd714091a4a"),
"lastName" : "test",
"firstName" : "test",
"email" : "test#test.com",
"password" : "$2a$10$wY9wr9oreza4fBX7CfXym.rPZUPrcesigYIfWd0zbM4dDjBy6k3vy",
"username" : "test",
"system" : {
"item" : 1,
"reading" : [ ]
},
"__v" : 0
}
I have followed
Mongodb $push in nested array
and this
Insert data in nested array in mongodb
and many more questions but cannot find whats wrong.
Since the positional $ operator acts as a placeholder for the first element that matches the query document, and the array field must appear as part of the query document, your update
operation does not satisfy these conditions hence it suffers from the error fate you are getting. In your query you are only referencing the "system.item" field which is not an array.
The other option you can do is ditch the positional $ operator in your update and just add the object to the array using $addToset which adds elements to an array only if they do not already exist in the set:
db.users.update(
{
"email": "test#test.com",
"system.item": 1
},
{
"$addToSet": {
"system.reading": {
"adc1" : "123",
"adc2": "1245",
"id":"1"
}
}
}
)
So i was trying to update data store in the mongodb database, using Mongoose
original data structure looks like this
{
"_id" : ObjectId("234u232kjrkjwebrkw"),
"local" : {
"password" : "sdflsdjflsdjlfkjsdlkfjklsdjflksd",
"email" : "email#email.com"
},
"__v" : 0
}
I'm trying to update the "userName" property in the "local". after which it supposed to be
{
"_id" : ObjectId("234u232kjrkjwebrkw"),
"local" : {
"password" : "sdflsdjflsdjlfkjsdlkfjklsdjflksd",
"email" : "email#email.com",
"userName" : "yowhatsup"
},
"__v" : 0
}
I used this
User.findByIdAndUpdate("54a490ab6e13cca1d47870d6", {local:{ userName: 'jasonBorne' }}, { upsert: true }, function(){})
it became this
{
"_id" : ObjectId("234u232kjrkjwebrkw"),
"local" : {
"userName" : "yowhatsup"
},
"__v" : 0
}
it's been overwritten.
how to avoid this?
note: the new "userName" is defined as {type:String,default:null} when the model was structred.
You can do an an update if exists, otherwise insert:
Arguments for update are: findQuery, data, queryOptions, onComplete
User.update({"_id" : ObjectId("234u232kjrkjwebrkw")}, { $set: { "local.userName": "whatsup" } }, { upsert: true }, function(err){...});