Regarding Needs to create an Index on MongoDB for $lookup query - node.js

I have 3 different collections which has build relation with the another field then primary id.
For ex, I have users, educations and user profiles 3 different collection and users and educations have relation with key user_id of user and same for userinfo. please find sample data.
let users = [
{ _id: ObjectId("1"), user_id: 1, name: "Nitin", is_active: true, is_deleted: false },
{ _id: ObjectId("2"), user_id: 2, name: "Vishal", is_active: true, is_deleted: false },
]
let educations =
[
{ _id: ObjectId("33"), user: 1, std: "10", is_active: true, is_deleted: false },
{ _id: ObjectId("44"), user: 2, std: "8", is_active: true, is_deleted: false },
]
let userinfo = [
{ _id: ObjectId("33"), user: 1, weight: "50", is_active: true, is_deleted: false },
{ _id: ObjectId("44"), user: 2, weight: "45", is_active: true, is_deleted: false },
]
Now I would like to fetch the relation data and created query by using $lookup by following way.
let criteria = {
is_active: true,
is_deleted: false,
};
const result = await mongoose.connection.db.collection('users').aggregate([
//education
{ $lookup: { from: "education", localField: "user_id", foreignField: "user", as: "education" } },
{ $unwind: { path: "$education", preserveNullAndEmptyArrays: true } },
//userinfo
{ $lookup: { from: "userinfo", localField: "user_id", foreignField: "user", as: "userinfo" } },
{ $unwind: { path: "$userinfo", preserveNullAndEmptyArrays: true } },
{ $match: { is_deleted: false } },
{ $match: criteria },
{
"$facet": {
"totalLocation": [
{ $match: criteria },
{ "$count": "count" },
],
}
},
{
"$project": {
"totalLocation": { "$arrayElemAt": ["$totalLocation.count", 0] },
}
}
], { allowDiskUse: true, collation: { locale: 'en_US', alternate: "shifted" } }).toArray();
This query works completely fine and return data as expected. But now as data are growing so this query became slower and we would make it faster. one of solution which I found that to create an index in a way so we can have faster result. I have tried but it doesn't works for me
so can anyone help me to create a index on this kind of query. or any another solution.
Explanation as requested in comment
{
"stages" : [
{
"$cursor" : {
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "mydata.users",
"indexFilterSet" : false,
"parsedQuery" : {
"is_deleted" : {
"$eq" : false
}
},
"queryHash" : "242D9E6F",
"planCacheKey" : "386E80BB",
"winningPlan" : {
"stage" : "PROJECTION_SIMPLE",
"transformBy" : {
"cb_id" : 1,
"user" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"is_deleted" : {
"$eq" : false
}
},
"direction" : "forward"
}
},
"rejectedPlans" : []
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 2328,
"executionTimeMillis" : 265,
"totalKeysExamined" : 0,
"totalDocsExamined" : 2355,
"executionStages" : {
"stage" : "PROJECTION_SIMPLE",
"nReturned" : 2328,
"executionTimeMillisEstimate" : 10,
"works" : 2357,
"advanced" : 2328,
"needTime" : 28,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 1,
"transformBy" : {
"cb_id" : 1,
"user" : 1,
"_id" : 0
},
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"is_deleted" : {
"$eq" : false
}
},
"nReturned" : 2328,
"executionTimeMillisEstimate" : 0,
"works" : 2357,
"advanced" : 2328,
"needTime" : 28,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 1,
"direction" : "forward",
"docsExamined" : 2355
}
}
}
},
"nReturned" : NumberLong(2328),
"executionTimeMillisEstimate" : NumberLong(20)
},
{
"$lookup" : {
"from" : "education",
"as" : "education",
"localField" : "user_id",
"foreignField" : "user",
"unwinding" : {
"preserveNullAndEmptyArrays" : true
}
},
"nReturned" : NumberLong(2328),
"executionTimeMillisEstimate" : NumberLong(136)
},
{
"$lookup" : {
"from" : "userinfo",
"as" : "userinfo",
"localField" : "user_id",
"foreignField" : "user",
"unwinding" : {
"preserveNullAndEmptyArrays" : true
}
},
"nReturned" : NumberLong(2328),
"executionTimeMillisEstimate" : NumberLong(264)
},
{
"$facet" : {
"totalLocation" : [
{
"$teeConsumer" : {},
"nReturned" : NumberLong(2328),
"executionTimeMillisEstimate" : NumberLong(264)
},
{
"$group" : {
"_id" : {
"$const" : null
},
"count" : {
"$sum" : {
"$const" : 1
}
}
},
"nReturned" : NumberLong(1),
"executionTimeMillisEstimate" : NumberLong(264)
},
{
"$project" : {
"count" : true,
"_id" : false
},
"nReturned" : NumberLong(1),
"executionTimeMillisEstimate" : NumberLong(264)
}
]
},
"nReturned" : NumberLong(1),
"executionTimeMillisEstimate" : NumberLong(264)
},
{
"$project" : {
"_id" : true,
"totalLocation" : {
"$arrayElemAt" : [
"$totalLocation.count",
{
"$const" : 0.0
}
]
}
},
"nReturned" : NumberLong(1),
"executionTimeMillisEstimate" : NumberLong(264)
}
],
"serverInfo" : {},
"ok" : 1.0
}
Thanks

Related

High load on simple Mongo find with indices

I have a mongoDB I'm querying using NodeJS (running mongoose).
In this particular case I'm querying a bunch of collections and pipe the data as CSV into archiverjs to create a zip file. So I have an incoming request, the data gets queried using mongoose and a mongo cursor, piped into a pipeline which will end in archiverjs respectively the http response delivering the zip file to the user.
async function getSortedQueryCursor(...) {
...
const query = MODEL_LOOKUP[fileType]
.find(reducer)
.sort({ [idString]: 'asc' });
return query.cursor();
}
async function getData(...) {
const cursor = await getSortedQueryCursor(...);
return cursor
.pipe(filter1Stream)
.pipe(filter2Stream)
.pipe(filter3Stream)
.pipe(csvStringifyStream);
}
router.post('/:scenarioId', async (request, response) => {
...
const archive = Archiver(...);
archive.pipe(response);
const result = await getData(...);
archive.append(stream, { name: filepath });
return archive.finalize();
}
As soon as a particular collection is in the game (the collection holds roughly 40 million documents) the query lasts very long (>15s) and I can see the mongo process on 100% CPU during that time. Even more surprising as the result set is empty (no documents matching the query).
It's a rather simple query:
items.find({ scenarioId: 'ckqf5ulg38gu208eecxlf95fc' }, { sort: { dataId: 1 }
I have indices on scenarioId and dataId. If I run the query on the shell it returns in 30ms.
An explain() results in:
[
{
"queryPlanner": {
"plannerVersion": 1,
"namespace": "data.items",
"indexFilterSet": false,
"parsedQuery": {
"scenarioId": {
"$eq": "ckqf5ulg38gu208eecxlf95fc"
}
},
"winningPlan": {
"stage": "SORT",
"sortPattern": {
"itemId": 1
},
"memLimit": 104857600,
"type": "simple",
"inputStage": {
"stage": "FETCH",
"inputStage": {
"stage": "IXSCAN",
"keyPattern": {
"scenarioId": 1
},
"indexName": "scenarioId_1",
"isMultiKey": false,
"multiKeyPaths": {
"scenarioId": []
},
"isUnique": false,
"isSparse": false,
"isPartial": false,
"indexVersion": 2,
"direction": "forward",
"indexBounds": {
"scenarioId": [
"[\"ckqf5ulg38gu208eecxlf95fc\", \"ckqf5ulg38gu208eecxlf95fc\"]"
]
}
}
}
},
"rejectedPlans": [
...
]
},
"executionStats": {
"executionSuccess": true,
"nReturned": 0,
"executionTimeMillis": 0,
"totalKeysExamined": 0,
"totalDocsExamined": 0,
"executionStages": {
"stage": "SORT",
"nReturned": 0,
"executionTimeMillisEstimate": 0,
"works": 3,
"advanced": 0,
"needTime": 1,
"needYield": 0,
"saveState": 0,
"restoreState": 0,
"isEOF": 1,
"sortPattern": {
"dataId": 1
},
"memLimit": 104857600,
"type": "simple",
"totalDataSizeSorted": 0,
"usedDisk": false,
"inputStage": {
"stage": "FETCH",
"nReturned": 0,
"executionTimeMillisEstimate": 0,
"works": 1,
"advanced": 0,
"needTime": 0,
"needYield": 0,
"saveState": 0,
"restoreState": 0,
"isEOF": 1,
"docsExamined": 0,
"alreadyHasObj": 0,
"inputStage": {
"stage": "IXSCAN",
"nReturned": 0,
"executionTimeMillisEstimate": 0,
"works": 1,
"advanced": 0,
"needTime": 0,
"needYield": 0,
"saveState": 0,
"restoreState": 0,
"isEOF": 1,
"keyPattern": {
"scenarioId": 1
},
"indexName": "scenarioId_1",
"isMultiKey": false,
"multiKeyPaths": {
"scenarioId": []
},
"isUnique": false,
"isSparse": false,
"isPartial": false,
"indexVersion": 2,
"direction": "forward",
"indexBounds": {
"scenarioId": [
"[\"ckqf5ulg38gu208eecxlf95fc\", \"ckqf5ulg38gu208eecxlf95fc\"]"
]
},
"keysExamined": 0,
"seeks": 1,
"dupsTested": 0,
"dupsDropped": 0
}
}
},
...
},
"serverInfo": {
...
"version": "4.4.6",
"gitVersion": "72e66213c2c3eab37d9358d5e78ad7f5c1d0d0d7"
},
...
}
]
It tells me (I'm not very experienced in interpreting those results) that the query is quite cheap: "executionTimeMillisEstimate": 0, as it's not running a document scan "docsExamined": 0,.
Next I connected to the mongo server and ran db.currentOp({"secs_running": {$gte: 5}}) to get some information from this side:
{
"type" : "op",
...
"clientMetadata" : {
"driver" : {
"name" : "nodejs|Mongoose",
"version" : "3.6.5"
},
"os" : {
"type" : "Linux",
"name" : "linux",
"architecture" : "x64",
"version" : "5.8.0-50-generic"
},
"platform" : "'Node.js v14.17.0, LE (unified)",
"version" : "3.6.5|5.12.3"
},
"active" : true,
"secs_running" : NumberLong(16),
"microsecs_running" : NumberLong(16661409),
"op" : "query",
"ns" : "data.items",
"command" : {
"find" : "items",
"filter" : {
"scenarioId" : "ckqf5ulg38gu208eecxlf95fc"
},
"sort" : {
"itemId" : 1
},
"projection" : {
},
"returnKey" : false,
"showRecordId" : false,
"lsid" : {
"id" : UUID("be3ce18b-5365-4680-b734-543d06418301")
},
"$clusterTime" : {
"clusterTime" : Timestamp(1625498044, 1),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : 0
}
},
"$db" : "data",
"$readPreference" : {
"mode" : "primaryPreferred"
}
},
"numYields" : 14701,
"locks" : {
"ReplicationStateTransition" : "w",
"Global" : "r",
"Database" : "r",
"Collection" : "r"
},
"waitingForLock" : false,
"lockStats" : {
"ReplicationStateTransition" : {
"acquireCount" : {
"w" : NumberLong(14702)
}
},
"Global" : {
"acquireCount" : {
"r" : NumberLong(14702)
}
},
"Database" : {
"acquireCount" : {
"r" : NumberLong(14702)
}
},
"Collection" : {
"acquireCount" : {
"r" : NumberLong(14702)
}
},
"Mutex" : {
"acquireCount" : {
"r" : NumberLong(1)
}
}
},
"waitingForFlowControl" : false,
"flowControlStats" : {
}
}
Any ideas how to improve the performance or find the bottleneck in my application? As the load is high on mongo side and no documents are found/passed to the application I guess it's mongo having trouble ...
EDIT: I've logged the whole process from DB side using db.setProfilingLevel(2) and db.system.profile.find().pretty(). Here we can see that the whole collection (or am I misinterpreting "docsExamined" : 39612167?) is queried:
{
"op" : "query",
"ns" : "data.items",
"command" : {
"find" : "items",
"filter" : {
"scenarioId" : "ckqf5ulg38gu208eecxlf95fc"
},
"sort" : {
"dataId" : 1
},
"projection" : {
},
...
"$db" : "data",
"$readPreference" : {
"mode" : "primaryPreferred"
}
},
"keysExamined" : 39612167,
"docsExamined" : 39612167,
"cursorExhausted" : true,
"numYield" : 39613,
"nreturned" : 0,
"queryHash" : "B7F40289",
"planCacheKey" : "BADED068",
"locks" : {
"ReplicationStateTransition" : {
"acquireCount" : {
"w" : NumberLong(39615)
}
},
"Global" : {
"acquireCount" : {
"r" : NumberLong(39615)
}
},
"Database" : {
"acquireCount" : {
"r" : NumberLong(39614)
}
},
"Collection" : {
"acquireCount" : {
"r" : NumberLong(39614)
}
},
"Mutex" : {
"acquireCount" : {
"r" : NumberLong(1)
}
}
},
"flowControl" : {
},
"storage" : {
},
"responseLength" : 242,
"protocol" : "op_msg",
"millis" : 48401,
"planSummary" : "IXSCAN { dataId: 1 }",
"execStats" : {
"stage" : "CACHED_PLAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 48401,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 39613,
"restoreState" : 39613,
"isEOF" : 1,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"scenarioId" : {
"$eq" : "ckqf5ulg38gu208eecxlf95fc"
}
},
"nReturned" : 0,
"executionTimeMillisEstimate" : 6270,
"works" : 39612168,
"advanced" : 0,
"needTime" : 39612167,
"needYield" : 0,
"saveState" : 39613,
"restoreState" : 39613,
"isEOF" : 1,
"docsExamined" : 39612167,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 39612167,
"executionTimeMillisEstimate" : 2151,
"works" : 39612168,
"advanced" : 39612167,
"needTime" : 0,
"needYield" : 0,
"saveState" : 39613,
"restoreState" : 39613,
"isEOF" : 1,
"keyPattern" : {
"dataId" : 1
},
"indexName" : "dataId_1",
"isMultiKey" : false,
"multiKeyPaths" : {
"dataId" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"dataId" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 39612167,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0
}
}
}
(As usual) it seems like the indices are not properly set. I've created a new (secondary?) index:
{
"dataId" : 1,
"scenarioId": 1
}
and now the query returns within milliseconds ...
EDIT: What still makes me wonder is that the shell query returned in milliseconds and the mongoose query took ages. Even though the queries seem to be identical (from my point of view) mongo treats them differently.

Mongoose aggregate sort and limit on a lookup

Is there any way to sort photo's by createdAt and then limit them to 500 in the aggregate?
const account = await userModel.aggregate([
{ $match: { 'shared.username': username } },
{
$lookup: {
as: 'photos',
foreignField: 'userId',
from: 'photos',
localField: '_id',
},
},
{
$project: {
_id: 1.0,
photos: 1.0,
shared: 1.0,
},
},
]);
the data
{
"_id" : ObjectId("5e10983cb182628af48e590a"),
"shared" : {
"currency" : "USD",
"followers" : 0,
"following" : 0,
"language" : "en",
"loggedIn" : true,
"twofactor" : false,
"warningMessage" : "verify",
"email" : "email#gmail.com",
"fullName" : "James",
"username" : "57fe31142e10",
"location" : "/57fe31142e10"
},
"photos" : [
{
"_id" : ObjectId("5e117b8f227a32597cdcbb6e"),
"category" : "Double Deckers",
"previewId" : "5e10983cb182628af48e590a/0f5d0010a4e55794419c03328184cfb45984bf43.jpg",
"published" : true,
"thumbnailId" : "5e10983cb182628af48e590a/54265d07a962f8544a9ebdc1d876775d9eeed471.jpg",
"userId" : ObjectId("5e10983cb182628af48e590a"),
"zoomId" : "5e10983cb182628af48e590a/fe0b2016a0be2b26259aaf5152ef22edfffa0c57.jpg",
"createdAt" : ISODate("2020-01-05T06:00:47.756+0000"),
"updatedAt" : ISODate("2020-01-05T06:00:47.756+0000"),
"__v" : 0
},
{
"_id" : ObjectId("5e11ab2f0f451779f70f89b1"),
"category" : "Single Decker",
"previewId" : "5e10983cb182628af48e590a/30d496e44faae1345e4c555accfdc6446fd11945.jpg",
"published" : true,
"thumbnailId" : "5e10983cb182628af48e590a/9293dd5517f694341a2e582df670dc9bbaba0763.jpg",
"userId" : ObjectId("5e10983cb182628af48e590a"),
"zoomId" : "5e10983cb182628af48e590a/0a038434e857b05ef8aa46da6dab01259ba2b03e.jpg",
"createdAt" : ISODate("2020-01-05T09:23:59.007+0000"),
"updatedAt" : ISODate("2020-01-05T09:23:59.007+0000"),
"__v" : 0
},
{
"_id" : ObjectId("5e11aba00f451779f70f89b2"),
"category" : "Midi",
"previewId" : "5e10983cb182628af48e590a/2267d14aa642d6cee86319909177ce4eef45cbcc.jpg",
"published" : true,
"thumbnailId" : "5e10983cb182628af48e590a/9d006f1ed361540ecf9d24c807a111770cf9e44f.jpg",
"userId" : ObjectId("5e10983cb182628af48e590a"),
"zoomId" : "5e10983cb182628af48e590a/88a87440a9b7845aa44c72411edddc98ea56f6b9.jpg",
"createdAt" : ISODate("2020-01-05T09:25:52.019+0000"),
"updatedAt" : ISODate("2020-01-05T09:25:52.019+0000"),
"__v" : 0
}
]
}
I still need the _id and shared array
You can use $lookup with custom pipeline and apply $sort with $limit:
const account = await userModel.aggregate([
{ $match: { 'shared.username': username } },
{
$lookup: {
as: 'photos',
let: { local_id: '$_id' },
pipeline: [
{ $match: { $expr: { $eq: [ '$$local_id', '$userId' ] } } },
{ $sort: { 'createdAt': 1 } },
{ $limit: 500 }
],
from: 'photos',
}
},
{
$project: {
_id: 1.0,
photos: 1.0,
shared: 1.0,
}
},
]);

comapare array of id in and get value through lookup mongoDB

my collection Stucture is like below:
{"_id" : ObjectId("5aec2ce5020ba15d2fb2665a"),
"password" : "e10adc3949ba59abbe56e057f20f883e",
"unique_id" : 22,
"first_name" : "Foram",
"last_name" : "Test ",
"country_phone_code" : "+61",
"email" : "a#a.com",
"phone" : "1231231231",
"device_type" : "android",
"admintypeid" : ObjectId("5a9f9b55b9e8fa66f9b934c0"),
"vehicle_detail" : [
{
"service_name" : "You",
"_id" : ObjectId("5aec2d08020ba15d2fb2665c"),
"name" : "Qee",
"plate_no" : "Qwe111",
"model" : "Qee",
"color" : "Blue",
"passing_year" : "2005",
"service_type" : [
ObjectId("5a9f9c53b9e8fa66f9b934c1")
],
"admin_type_id" : ObjectId("5a9f9b55b9e8fa66f9b934c0"),
"is_selected" : true,
"pictureData" : "vehicle_images/5aec2d08020ba15d2fb2665cWjyP.jpg"
},
{
"pictureData" : "vehicle_images/5aec35e00efc106080e14ec7y2tO.jpg",
"is_selected" : false,
"admin_type_id" : ObjectId("5a9f9b55b9e8fa66f9b934c0"),
"service_type" : [
ObjectId("5a9f9c53b9e8fa66f9b934c1"),
ObjectId("5ac954b7075e16583dc3311f")
],
"passing_year" : "2018",
"color" : "No",
"model" : "Bcs",
"plate_no" : "12112",
"name" : "Bcs",
"_id" : ObjectId("5aec35e00efc106080e14ec7"),
"service_name" : "You"
}
],
"service_type" : [
ObjectId("5a9f9c53b9e8fa66f9b934c1")
],
"is_vehicle_document_uploaded" : true,
"is_trip" : [],
"__v" : 14
}
i want to campare ids in my service_type array which is in vehicle_detail array
to another collection named city_types than from that i have to comapare typid of city_types collection to tyes collection and get value of name from type collection
my city_type collection would be something like below
{
"_id" : ObjectId("5a9f9c53b9e8fa66f9b934c1"),
"countryid" : ObjectId("5abb275fd20731597cc01229"),
"cityid" : ObjectId("5abb27cbd20731597cc0122a"),
"typeid" : ObjectId("5a9f9b55b9e8fa66f9b934c0")
"__v" : 0}
containing type_id
and my types collection is like below
{
"_id" : ObjectId("5a9f9b55b9e8fa66f9b934c0"),
"typename" : "You",
"description" : "KARRU You",
"__v" : 0,
"main_type" : 0
}
what i want it name of type
from id
i have used code below:
exports.get_provider_vehicle_list = function (req, res, next) {
var mongoose = require('mongoose');
var Schema = mongoose.Types.ObjectId;
var condition = {$match: {"_id": Schema(req.body.provider_id)}};
var vunwind = {$unwind: "$vehicle_detail"}
var lookup = {
$lookup:
{
from: "types",
localField: "vehicle_detail.admin_type_id",
foreignField: "_id",
as: "type_detail"
}
};
var unwind = {$unwind: {
path: "$type_detail",
preserveNullAndEmptyArrays: true
}
};
var group = {$group: {
_id: null,
"vehicle_detail": {$push: {
is_selected: "$vehicle_detail.is_selected",
admin_type_id: "$vehicle_detail.admin_type_id",
service_type: "$vehicle_detail.service_type",
passing_year: "$vehicle_detail.passing_year",
color: "$vehicle_detail.color",
model: "$vehicle_detail.model",
plate_no: "$vehicle_detail.plate_no",
name: "$vehicle_detail.name",
_id: "$vehicle_detail._id",
type_image_url: '$type_detail.type_image_url',
service_name:"$vehicle_detail.service_name",
pictureData:"$vehicle_detail.pictureData"
}}
}
}
Provider.aggregate([condition, vunwind, lookup, unwind, group], function (err, provider) {
if (err || provider.length == 0) {
res.json({success: true, vehicle_list: []})
} else {
res.json({success: true, vehicle_list: provider[0].vehicle_detail})
}
})
};
and my res is like below:
{
"success": true,
"vehicle_list": [
{
"is_selected": true,
"admin_type_id": "5a9f9b55b9e8fa66f9b934c0",
"service_type": [
"5a9f9c53b9e8fa66f9b934c1",
"5ac954b7075e16583dc3311f"
],
"passing_year": "2005",
"color": "Blue",
"model": "Qee",
"plate_no": "Qwe111",
"name": "Qee",
"_id": "5aec2d08020ba15d2fb2665c",
"type_image_url": "service_type_images/5a9f9b55b9e8fa66f9b934c0mjYA.png",
"service_name": "You",
"pictureData": "vehicle_images/5aec2d08020ba15d2fb2665cWjyP.jpg"
}
]
}
and res i want is below
{
"success": true,
"vehicle_list": [
{
"is_selected": true,
"admin_type_id": "5a9f9b55b9e8fa66f9b934c0",
"service_type": [
"5a9f9c53b9e8fa66f9b934c1",
"5ac954b7075e16583dc3311f"
],
"passing_year": "2005",
"color": "Blue",
"model": "Qee",
"plate_no": "Qwe111",
"name": "Qee",
"_id": "5aec2d08020ba15d2fb2665c",
"type_image_url": "service_type_images/5a9f9b55b9e8fa66f9b934c0mjYA.png",
"service_type_name": [
"You"
],
"service_name": "You",
"pictureData": "vehicle_images/5aec2d08020ba15d2fb2665cWjyP.jpg"
}
]
}
please check.
db.getCollection('vehicle').aggregate([
{
$unwind: {
path: '$vehicle_detail',
preserveNullAndEmptyArrays: true,
},
},
{
$unwind: {
path: '$vehicle_detail.service_type',
preserveNullAndEmptyArrays: true,
},
},
{
$lookup: {
from: 'city_type',
localField: 'vehicle_detail.service_type',
foreignField: '_id',
as: 'city_type',
},
},
{
$unwind: {
path: '$city_type',
preserveNullAndEmptyArrays: true,
},
},
{
$lookup: {
from: 'types',
localField: 'city_type.typeid',
foreignField: '_id',
as: 'types',
},
},
{
$unwind: {
path: '$types',
preserveNullAndEmptyArrays: false,
},
},
{
$group: {
_id: '$_id',
first_name: { $first: 1 },
last_name: { $first: 1 },
types: { $first: '$types' },
},
},
])
Result
{
"_id" : ObjectId("5aec2ce5020ba15d2fb2665a"),
"first_name" : 1.0,
"last_name" : 1.0,
"types" : {
"_id" : ObjectId("5a9f9b55b9e8fa66f9b934c0"),
"typename" : "You",
"description" : "KARRU You",
"__v" : 0,
"main_type" : 0
}
}
Note : You can added fields in group which you need.
I am sending what I understood from your explanation. I hope it helps.
Some points:
I changed your objects city_types and types to an array, since you said Collections;
I link the id to match Vehicle->city->types
function ObjectId(id){
return id;
}
var obj = {"_id" : ObjectId("5aec2ce5020ba15d2fb2665a"),
"password" : "e10adc3949ba59abbe56e057f20f883e",
"unique_id" : 22,
"first_name" : "Foram",
"last_name" : "Test ",
"country_phone_code" : "+61",
"email" : "a#a.com",
"phone" : "1231231231",
"device_type" : "android",
"admintypeid" : ObjectId("5a9f9b55b9e8fa66f9b934c0"),
"vehicle_detail" : [
{
"service_name" : "Service 1",
"_id" : ObjectId("5aec2d08020ba15d2fb2665c"),
"name" : "Qee",
"plate_no" : "Qwe111",
"model" : "Qee",
"color" : "Blue",
"passing_year" : "2005",
"service_type" : [
ObjectId("5a9f9c53b9e8fa66f9b934c1")
],
"admin_type_id" : ObjectId("5a9f9b55b9e8fa66f9b934c0"),
"is_selected" : true,
"pictureData" : "vehicle_images/5aec2d08020ba15d2fb2665cWjyP.jpg"
},
{
"pictureData" : "vehicle_images/5aec35e00efc106080e14ec7y2tO.jpg",
"is_selected" : false,
"admin_type_id" : ObjectId("5a9f9b55b9e8fa66f9b934c0"),
"service_type" : [
ObjectId("5a9f9c53b9e8fa66f9b934c1"),
ObjectId("5ac954b7075e16583dc3311f")
],
"passing_year" : "2018",
"color" : "No",
"model" : "Bcs",
"plate_no" : "12112",
"name" : "Bcs",
"_id" : ObjectId("5aec35e00efc106080e14ec7"),
"service_name" : "Service 2"
}
],
"service_type" : [
ObjectId("5a9f9c53b9e8fa66f9b934c1")
],
"is_vehicle_document_uploaded" : true,
"is_trip" : [],
"__v" : 14
};
var city_types = [{
"_id" : ObjectId("5aec2d08020ba15d2fb2665c"),
"countryid" : ObjectId("5abb275fd20731597cc01229"),
"cityid" : ObjectId("5abb27cbd20731597cc0122a"),
"typeid" : ObjectId("5a9f9b55b9e8fa66f9b934c0"),
"__v" : 0
}];
var types = [{
"_id" : ObjectId("5a9f9b55b9e8fa66f9b934c0"),
"typename" : "You",
"description" : "KARRU You",
"__v" : 0,
"main_type" : 0
}];
function print() {
//iterate over vehicle_detail
obj.vehicle_detail.forEach((vehicle)=>{
console.log("vehicle="+ vehicle._id);
//iterate over cities type
city_types.forEach((city)=> {
console.log("city="+ city._id);
if (city._id == vehicle._id) {
//iterate over types
types.forEach((type)=> {
console.log("type="+ type._id);
if (city.typeid == type._id) {
console.log("Type Name: "+type.typename);
}
})
}
})
});
}
print();
The output of the program is:
>node index4.js
vehicle=5aec2d08020ba15d2fb2665c
city=5aec2d08020ba15d2fb2665c
type=5a9f9b55b9e8fa66f9b934c0
Type Name: You
vehicle=5aec35e00efc106080e14ec7
city=5aec2d08020ba15d2fb2665c

How can I use mongoose $lookup operator for lookup chaining and then grouping the result?

Lets consider I have three pseudo collections like following
users:
{ "_id" : ObjectId("5a2a2c3dcf7961800e44dfc9"), "uid" : 1234,
"name" : "ashish", "email" : "info#gmail.com" },
{ "_id" : ObjectId("5a2a2c51cf7961800e44dfca"), "uid" : 1235,
"name" : "swapnil", "email" : "info#femail.com" }
bank_details:
{ "_id" : ObjectId("5a2a2c9bcf7961800e44dfcb"), "uid" : 1234, "acc_no" : 1111 },
{ "_id" : ObjectId("5a2a2cc9cf7961800e44dfcc"), "uid" : 1234, "acc_no" : 2222 },
{ "_id" : ObjectId("5a2a2cd7cf7961800e44dfcd"), "uid" : 1235, "acc_no" : 3333 },
{ "_id" : ObjectId("5a2a2cdccf7961800e44dfce"), "uid" : 1235, "acc_no" : 4444 }
bank
{ "_id" : ObjectId("5a2a2d05cf7961800e44dfcf"), "acc_no" : 1111, "balance" : 100 },
{ "_id" : ObjectId("5a2a2d10cf7961800e44dfd0"), "acc_no" : 2222, "balance" : 200 },
{ "_id" : ObjectId("5a2a2d19cf7961800e44dfd1"), "acc_no" : 3333, "balance" : 300 },
{ "_id" : ObjectId("5a2a2d21cf7961800e44dfd2"), "acc_no" : 4444, "balance" : 400 }
I wanna have output as following
{ "_id" : ObjectId("5a2a2c3dcf7961800e44dfc9"), "uid" : "1234",
"name" : "ashish", "email" : "info#gmail.com" ,
bank_details: [{
acc_no:'1111',
balance:'100'
},{
acc_no:'2222',
balance:'200'
}]}
for each profile of user, I'm sure this can be done using $lookup and $group operator but I could never achieve it, I have tried to do this using several ways. Can anybody please write a sample query for the required result.
I'm trying to write the query using mongoose in NodeJs
You can do that using two lookups and grouping. Check out the following query:
db.getCollection('users').aggregate([{
$lookup:
{
from: "bank_details",
localField: "uid",
foreignField: "uid",
as: "bank_details"
}
},
{ $unwind:{ path: "$bank_details", preserveNullAndEmptyArrays: true }},
{
$lookup:{
from: "bank",
localField: "bank_details.acc_no",
foreignField: "acc_no",
as: "banks"
}
},
{ $unwind: { path: "$banks", preserveNullAndEmptyArrays: true } },
{
$group: {
_id: "$_id",
uid: {$first: "$uid"},
name: {$first: "$name"},
email: {$first: "$email"},
bank_details: {$push: {acc_no: "$banks.acc_no", balance: "$banks.balance"}}
}
},
{
"$project": {
uid: 1,
name: 1,
email: 1,
bank_details: { "$setDifference": [ "$bank_details", [{}, null] ] }
}
}
])
Results:
/* 1 */
{
"_id" : ObjectId("5a2a2c51cf7961800e44dfca"),
"uid" : 1235.0,
"name" : "swapnil",
"email" : "info#femail.com",
"bank_details" : [
{
"acc_no" : 3333.0,
"balance" : 300.0
},
{
"acc_no" : 4444.0,
"balance" : 400.0
}
]
}
/* 2 */
{
"_id" : ObjectId("5a2a2c3dcf7961800e44dfc9"),
"uid" : 1234.0,
"name" : "ashish",
"email" : "info#gmail.com",
"bank_details" : [
{
"acc_no" : 1111.0,
"balance" : 100.0
},
{
"acc_no" : 2222.0,
"balance" : 200.0
}
]
}
If you don't want to lose the users that have no bank_details you have to add a $project before you unwind. By "$unwind" you flatten the bank_details array and the documents that have an empty array, disappear after since there is nothing to unwind. So you can check your bank_details array if it's empty and add zero values to the fields:
db.getCollection('users').aggregate([{
$lookup: {
from: "bank_details",
localField: "uid",
foreignField: "uid",
as: "bank_details"
}
},
{
$project: {
"_id": "$_id",
uid: "$uid",
name: "$name",
email: "$email",
bank_details: {
$cond: [{
$eq: ["$bank_details", []]
},
[{
_id: 0,
uid: {
$literal: NumberInt(0)
},
acc_no: {
$literal: NumberInt(0)
}
}], '$bank_details'
]
}
}
},
{
$unwind: "$bank_details"
},
{
$lookup: {
from: "bank",
localField: "bank_details.acc_no",
foreignField: "acc_no",
as: "banks"
}
},
{
$project: {
"_id": "$_id",
uid: "$uid",
name: "$name",
email: "$email",
bank_details: "$bank_details",
banks: {
$cond: [{
$eq: ["$banks", []]
},
[{
_id: 0,
acc_no: {
$literal: NumberInt(0)
},
balance: {
$literal: NumberInt(0)
}
}], '$banks'
]
}
}
},
{
$unwind: "$banks"
},
{
$group: {
"_id": "$_id",
uid: {
$first: "$uid"
},
name: {
$first: "$name"
},
email: {
$first: "$email"
},
bank_details: {
$push: {
acc_no: "$banks.acc_no",
balance: "$banks.balance"
}
}
}
}
])
Here the results of the query:
/* 1 */
{
"_id" : ObjectId("5a2a83a53b3899dec93be178"),
"uid" : 1239,
"name" : "emil",
"email" : "emil#femail.com",
"bank_details" : [
{
"acc_no" : 0.0,
"balance" : 0.0
}
]
}
/* 2 */
{
"_id" : ObjectId("5a2a2c51cf7961800e44dfca"),
"uid" : 1235,
"name" : "swapnil",
"email" : "info#femail.com",
"bank_details" : [
{
"acc_no" : 3333,
"balance" : 300
},
{
"acc_no" : 4444,
"balance" : 400
}
]
}
/* 3 */
{
"_id" : ObjectId("5a2a2c3dcf7961800e44dfc9"),
"uid" : 1234,
"name" : "ashish",
"email" : "info#gmail.com",
"bank_details" : [
{
"acc_no" : 1111,
"balance" : 100
},
{
"acc_no" : 2222,
"balance" : 200
}
]
}

Search in subdocument

Hello I want to get all the contacts with syncFlag true in contact object.
I tried this solution but only returns only one sub document
and I also tried this solution it returns all the documents either they are matched or not
How to find document and single subdocument matching given criterias in MongoDB collection
Here is the sample document
{
"_id" : ObjectId("57ce7d6c7387d533bfa2d45c"),
"ITBCompanyId" : 2608,
"updatedAt" : ISODate("2016-09-06T12:19:35.972Z"),
"createdAt" : ISODate("2016-09-06T08:25:16.325Z"),
"name" : "This is test",
"identifier" : "THDNOM2",
"addressLine1" : "Valencia Lahore",
"syncFlag" : true,
"orgId" : "1",
"deletedAt" : null,
"whois" : [
{
"test" : "noman"
}
],
"configuration" : [
{
"test" : "noman"
}
],
"contact" : [
{
"firstName" : "Active",
"_id" : ObjectId("57ceb04811f005420b7ed54a"),
"syncFlag" : false,
"communicationItems" : [],
"customFields" : []
},
{
"firstName" : "Active",
"_id" : ObjectId("57ceb04811f005420b7ed54b"),
"syncFlag" : false,
"communicationItems" : [],
"customFields" : []
},
{
"firstName" : "Active",
"_id" : ObjectId("57ceb44b5f8b534bc312aacd"),
"syncFlag" : true,
"communicationItems" : [],
"customFields" : []
},
{
"firstName" : "Active",
"_id" : ObjectId("57ceb457f141fd4c1c98a748"),
"syncFlag" : true,
"communicationItems" : [],
"customFields" : []
}
],
"agreement" : [
{
"test" : "noman"
}
],
"companySite" : [
{
"test" : "noman"
}
],
"companyNote" : [
{
"test" : "noman"
}
],
"type" : {
"name" : "Client"
},
"status" : {
"name" : "Active"
},
"id" : "19493",
"__v" : 0,
"_info" : {
"updatedBy" : "Omer",
"lastUpdated" : ISODate("2016-09-06T11:52:07.000Z")
}
}
Expected Output:
ITBCompanyId: 1,
contact: [{
"firstName" : "Active",
"_id" : ObjectId("57ceb04811f005420b7ed54b"),
"syncFlag" : true,
"communicationItems" : [],
"customFields" : []
}]
Try this. You should get the document as required. Below I have used an aggregate query to format the mongodb response as required. For more information regarding mongodb aggregate, you can refer this. I reffered this stack overflow post for forming this query.
db.companies.aggregate(
{ $match: {"contact.syncFlag": true }},
{ $unwind: '$contact'},
{ $match: {"contact.syncFlag": true }},
{ $group: {
_id: '$_id',
contact: {$push: '$contact'},
}
});
If you want all the other fields included in the document. You can try this.
db.companies.aggregate(
{ $match: {"contact.syncFlag": true }},
{ $unwind: '$contact'},
{ $match: {"contact.syncFlag": true }},
{ $group: {
_id: '$_id',
ITBCompanyId: {$first: '$ITBCompanyId'},
updatedAt: {$first: '$updatedAt'},
createdAt: {$first: '$createdAt'},
name: {$first: '$name'},
identifier: {$first: '$identifier'},
addressLine1: {$first: '$addressLine1'},
syncFlag: {$first: '$syncFlag'},
orgId: {$first: '$orgId'},
deletedAt: {$first: '$deletedAt'},
whois: {$first: '$whois'},
configuration: {$first: '$configuration'},
contact: {$push: '$contact'},
agreement: {$first: '$agreement'},
companySite: {$first: '$companySite'},
companyNote: {$first: '$companyNote'},
type: {$first: '$type'},
status: {$first: '$status'},
id: {$first: '$id'},
_info: {$first: '$_info'},
}
});

Resources