Empty array output from the $lookup aggregation operator - node.js

I'm trying to get the most popular recipes.
I have 2 relevant collections: recipes and favoriterecipes.
Things to consider: I've double checked collections names and ids.
In the db itself the _recipe field in favoriterecipes is type string and in recipes it is an ObjectId. (maybe a type conversion is required? even though I didn't see such thing in "lookup" examples).
favoriteRecipe.js:
const mongoose = require("mongoose");
const Recipe = require("./recipe");
const User = require("./user");
const FavoritesRecipesSchema = new mongoose.Schema({
_recipe: { type: mongoose.Schema.Types.ObjectId, ref: Recipe },
_user: { type: mongoose.Schema.Types.ObjectId, ref: User },
});
module.exports = mongoose.model("FavoriteRecipe", FavoritesRecipesSchema);
recipe.js:
const mongoose = require("mongoose");
const User = require("./user");
const RecipeScheme = new mongoose.Schema({
name: String,
ingredients: [String],
instructions: String,
image: String,
date: { type: Date, default: Date.now },
tags: [String],
_user: { type: mongoose.Schema.Types.ObjectId, ref: User },
});
module.exports = mongoose.model("Recipe", RecipeScheme);
controller.js:
exports.popular = async function (req, res, next) {
try {
const popular_recipes = await favoriteRecipe.aggregate([
{
$group: {
_id: "$_recipe",
recipeCount: { $sum: 1 },
},
},
{ $sort: { recipeCount: -1 } },
{
$lookup: {
from: "recipes",
localField: "_id",
foreignField: "_id",
as: "recipe",
},
},
// { $unwind: "$recipe" },
// {
// $project: {
// _id: "$recipe",
// recipeCount: 1,
// },
// },
]);
res.json(popular_recipes);
} catch (error) {
next(error);
}
};
response output:
[
{
"_id": "6053349353b5f5632986b2c2",
"recipeCount": 3,
"recipe": []
},
{
"_id": "6053349353b5f5632986b2c3",
"recipeCount": 2,
"recipe": []
},
{
"_id": "605603945b4aeb0d2458153e",
"recipeCount": 1,
"recipe": []
}
]

Eventually I found that I have to convert the id string to an object id. the solution:
exports.popular = async function (req, res, next) {
try {
const popular_recipes = await favoriteRecipe.aggregate([
{
$group: {
_id: { $toObjectId: "$_recipe" },
recipeCount: { $sum: 1 },
},
},
{ $sort: { recipeCount: -1 } },
{
$lookup: {
from: "recipes",
localField: "_id",
foreignField: "_id",
as: "recipe",
},
},
{ $unwind: "$recipe" },
{
$project: {
_id: "$recipe",
recipeCount: 1,
},
},
]);
res.json(popular_recipes);
} catch (error) {
next(error);
}
};

Related

Mongoose find by field in inner object (ObjectID)

For example I have two models
const User = new Schema({
name: { type: String },
});
const Message = new Schema({
user: { type: ObjectId, ref: 'User' },
message: { type: String },
});
How to find messages by the key "name" in the user?
It doesn't work
exports.get = async (req, res) => {
return Message.find({ "user.name": req.query.name })
.populate('user', 'name')
.then((data) => res.json(data));
}
I understand why "user.name" doesn't work, but I don't know how to solve it
You would want to use $lookup -> unwind -> match:
Unwind: convert user array to user object from look up stage
exports.get = async (req, res) => {
return Message.aggregate([
{
$lookup: {
from: "users",
localField: "user",
foreignField: "_id",
as: "user",
},
},
{
$unwind: {
path: "$user",
},
},
{
$match: {
"user.name": req.query.name,
},
},
])
.then((data) => res.json(data));
}
There is no concept of a join in Mongodb. Hence I would recommend using straight schema definition like this:
const User = new Schema({
name: { type: String },
});
const Message = new Schema({
user: [User],
message: { type: String },
});
then you wouldn't need to use populate anymore. Instead, you would have something like this:
Message.find({ "user.name": req.query.name })
However, If you still prefer to use the current approach, You can try this:
Message.find()
.populate({
path: "user",
match: {
name: req.query.name
}
})
...
Basically you need the correct ordering of table
User must be your primary table and Messages must be your secondary table from which you should lookup.
The query is as following;
exports.get = async (req, res) => {
const queryLength = req.query.name ? req.query.name.length : 0;
return User.aggregate([
{
$match: {
name: {
"$regex": req.query.name,
"$options": '-i'
}
},
$lookup: {
let: { id: _id },
from: 'Messages',
pipeline: [
{
$match: {
$expr: { $eq: ["$user", "$$id"] }
}
},
{
$project: {
user: 0,
_id: 0,
}
}
],
as: "Messages"
},
$project: {
regex_match: {
$eq: [
req.query.name,
{ $substr: [{ $toLower: "$name" }, 0, queryLength] }
]
},
Messages: 1
},
$sort: {
regex_match: -1//we use regex match to get the most match result on the top like if u search "Al" then all results with al in them will be on top
},
$project: {
Messages: 1,
regex_match: 0
}
}
])
.then((data) => res.json(data));
}
This will return all the messages as an array with field name messages of "name" person e.g;
{
Messages: [{_id: 1234, message: "Hello"},{_id: 12345, message: "Hi"} ]
}

mongoose middleware, show length of array in model

I want to get the size of array in the model:
const UserSchema = mongoose.Schema(
{
username: { type: String, lowercase: true, required: true },
items: { type: [mongoose.Types.ObjectId], default: [] },
},
{
toJSON : {
virtuals : true
},
timestamps: true,
}
);
UserSchema.virtual("itemsCount").get(function () {
return this.items.length;
});
module.exports = {
UserModel: mongoose.model("user", UserSchema ),
};
const ProductSchema = mongoose.Schema(
{
name: { type: String, required: true },
owner: { type: mongoose.Types.ObjectId,required: true, ref:"user"
},
},
{
toJSON : {
virtuals : true
},
timestamps: true,
}
);
module.exports = {
ProductModel: mongoose.model("product", ProductSchema ),
};
But I want to hide items in the output whenever I try to use projection it gives an error:
Cannot read properties of undefined (reading 'length')
const newPosts = await ProductModel.find({}).populate([{ path: "owner", select: { itemsCount: 0 }}]);
if I don't use select it works:
const newPosts = await ProductModel.find({}).populate([{ path: "owner" }}]);
But I don't want to show items filed in output
You can use aggregation pipeline for this:
ProductModel.aggregate([
{
"$lookup": {
"from": "users",
"localField": "user",
"foreignField": "_id",
"as": "users"
}
},
{
"$unwind": "$users"
},
{
"$project": {
_id: "$_id",
name: "$name",
"users": {
itemsCount: {
$size: "$users.items"
}
}
}
}
])
Read more about $lookup, $unwind, $project to understand.
Here is the Mongodb playground to see the results: https://mongoplayground.net/p/R0ZQiV8I-YM

Join two or more queries in mongo db node.js and get result as a single object using aggregate query

I have two collections as follows:
import mongoose from "mongoose";
const projectSchema = mongoose.Schema({
id: String,
userId: { type: mongoose.Schema.Types.ObjectId, ref: "User" },
title: String,
details: String,
location: String,
rate: String,
status: {
type: String,
default: "active",
},
createdAt: {
type: Date,
default: new Date(),
},
});
const Project = mongoose.model("Project", projectSchema);
export default Project;
import mongoose from "mongoose";
const proposalSchema = mongoose.Schema({
id: String,
userId: { type: mongoose.Schema.Types.ObjectId, ref: "User" },
projectId: { type: mongoose.Schema.Types.ObjectId, ref: "Project" },
rate: String,
message: String,
createdAt: {
type: Date,
default: new Date(),
},
});
const Proposal = mongoose.model("Proposal", proposalSchema);
export default Proposal;
And in response to a GET request, I want to get all the projects which are active and user has not sent the proposal to them, GET request will have the id of user.
(Proposal: When a user sends a proposal, a proposal object is created in proposals collections which has userId and ProjectId)
I have make it work using the below queries but it doesn't looks efficient and good. Is there a way I can get this result using aggregate query or any better way from this?
And also how I can efficiently can convert objectId to string Id here.
export const getProjects = async (req, res) => {
try {
const activeProjects = await Project.find({ status: "active" }, { _id: 1 });
const projectsWithProposals = await Proposal.find(
{
$and: [
{ userId: req.query.id },
{ projectId: { $in: activeProjects } },
],
},
{ _id: 0, projectId: 1 }
);
const stringsIds = projectsWithProposals.map((id) =>
id.projectId.toString()
);
const projects = await Project.find({
$and: [{ status: "active" }, { _id: { $nin: stringsIds } }],
});
res.status(200).json(projects);
} catch (error) {
res.status(404).json({ message: error.message });
}
};
Here is a aggregation function which delivers all Projects which have no proposal from a given user:
function getQ (userId) {
return [
{
"$match": {
"$expr": {
"$eq": [
"$status",
"active"
]
}
}
},
{
"$lookup": {
"from": "proposals",
"localField": "_id",
"foreignField": "projectId",
"as": "proposals"
}
},
{
"$set": {
"uids": "$proposals.userId"
}
},
{
"$unset": "proposals"
},
{
"$match": {
"$expr": {
"$not": [
{
"$in": [
mongoose.Types.ObjectId(userId),
"$uids"
]
}
]
}
}
},
{
"$unset": "uids"
},
{
"$limit": 10
}
]
}
db.Project.aggregate(getQ("62a61df204f2ce244ce0ffcc")) // input is user._id
.then(console.log)
.catch(console.error)
I have used the standard mongoose _ids so you might have to adapt the code if required.
The query does only output the Project collection data, although it would be easy to include other data as well.
Beware that limit is constant here. You could also convert skip and limit to function paramters which would make the function much more flexible if you are working with huge amounts of results.

How to get array of objects from current collection and data from another collection in same query

I have a users collection. Each user can have multiple friends whose _id values are stored in an array in their user document.
I want to render a list which has a user's list of friend names and under each individual name I want to have a location but the locations are in another document because each user can have multiple locations with no limit on the number, like their visited/favourite places.
These are the 2 queries in isolation but I need to somehow combine them.
This is the query which looks at the user ID values in the user's array of friends and then gets the relevant user info based on the ID.
friends = await User.findOne({ _id: userId })
.populate("friends", "name mobile", null, { sort: { name: 1 } })
.exec();
Then, to get the places for a particular user:
const place = await Place.find({ creator: userId });
So, I basically want to list the friends in a loop, each with their places under their name like:
Joe Soap
- London Bridge
- Eiffel Tower
Bob Builder
- Marienplatz
The data in mongoDb look like this:
Users:
{
"_id": {
"$oid": "5f2d9ec5053d4a754d6790e8"
},
"friends": [{
"$oid": "5f2da51e053e4a754d5790ec"
}, {
"$oid": "5f2da52e053d4a754d6790ed"
}],
"name": "Bob",
"email": "bob#gmail.com",
"created_at": {
"$date": "2020-08-07T18:34:45.781Z"
}
}
Places:
{
"_id": {
"$oid": "5f3695d79864bd6c7c94e38a"
},
"location": {
"latitude": -12.345678,
"longitude": 12.345678
},
"creator": {
"$oid": "5f2da51e053e4a754d5790ec"
},
}
The first join is basically getting data from the array inside the same user collection. The second one is getting data from another collection, places.
UPDATE: almost working
friends = await User.aggregate([
{ $match: { _id: new mongoose.Types.ObjectId(userId) } },
{
$lookup: {
from: "users",
localField: "friends",
foreignField: "_id",
as: "friends_names",
},
},
{ $unwind: "$friends_names" },
{
$lookup: {
from: "places",
localField: "friends",
foreignField: "creator",
as: "friends_places",
},
},
{ $unwind: "$friends_places" },
{
$project: {
"friends_names.name": 1,
"friends_places.saved_at": 1,
},
},
]);
Data returned:
[
{
_id: 5f2d9ec5053d4a754d6790e8,
friends_names: { name: 'Bob' },
friends_places: { saved_at: 2020-08-17T13:40:28.334Z }
},
{
_id: 5f2d9ec5053d4a754d6790e8,
friends_names: { name: 'Natalie' },
friends_places: { saved_at: 2020-08-17T13:40:28.334Z }
}
]
Edit Okay, I believe I have reverse engineered your minimal schema correctly:
const mongoose = require("mongoose");
mongoose.connect("mongodb://localhost/test", { useNewUrlParser: true });
mongoose.set("debug", true);
const db = mongoose.connection;
db.on("error", console.error.bind(console, "connection error:"));
db.once("open", async function () {
await mongoose.connection.db.dropDatabase();
// we're connected!
console.log("Connected");
const userSchema = new mongoose.Schema({
friends: [{ type: mongoose.Schema.Types.ObjectId, ref: "User" }],
name: String,
});
const placesSchema = new mongoose.Schema({
latitude: String,
creator: { type: mongoose.Schema.Types.ObjectId, ref: "User" },
});
const User = mongoose.model("User", userSchema);
const Place = mongoose.model("Place", placesSchema);
const bob = new User({ name: "Bob", friends: [] });
await bob.save();
const natalie = new User({ name: "Natalie", friends: [bob] });
await natalie.save();
//const chris = new User({ name: "Chris", friends: [] });
//await chris.save();
const john = new User({ name: "John", friends: [natalie, bob] });
await john.save();
const place1 = new Place({ latitude: "Place1", creator: bob });
const place3 = new Place({ latitude: "Place3", creator: bob });
const place2 = new Place({ latitude: "Place2", creator: natalie });
await place1.save();
await place2.save();
await place3.save();
await User.find(function (err, users) {
if (err) return console.error(err);
console.log(users);
});
await Place.find(function (err, places) {
if (err) return console.error(err);
//console.log(places);
});
var cc = await mongoose
.model("User")
.aggregate([
{ $match: { _id: john._id } },
{ $unwind: "$friends" },
{
$lookup: {
from: "places",
localField: "friends",
foreignField: "creator",
as: "friends_places",
}
}, { $lookup: {from: 'users', localField: 'friends', foreignField: '_id', as: 'friend_name'} },//, { $include: 'friends' }
{ $unwind: "$friends_places" }, { $unwind: "$friend_name" }//, { $skip: 1}, {$limit: 1}
])
.exec();
console.log(cc);
});
This is the most relevant part:
var cc = await mongoose
.model("User")
.aggregate([
{ $match: { _id: john._id } },
{ $unwind: "$friends" },
{
$lookup: {
from: "places",
localField: "friends",
foreignField: "creator",
as: "friends_places",
}
}, { $lookup: {from: 'users', localField: 'friends', foreignField: '_id', as: 'friend_name'} },//, { $include: 'friends' }
{ $unwind: "$friends_places" }, { $unwind: "$friend_name" }//, { $skip: 1}, {$limit: 1}
])
.exec();
The first unwind: friends 'flattens' the collections initially. So, basically we've got 'userId | friendId' for each user and friend. Then, for each row, we simply look up places created by him ($lookup). Finally, we unwind friends_places because we don't want them to be rendered as [object] in console output. Additionally, there this $match, because we only want to check one user's friends' places. Considering we want to know friend's name as well, we have to do one more join - this is why there's this second $lookup. After that a simple $unwind to get friend's detail and that's it.
Code yields the following:
[ { _id: 5f3aa3a9c6140e3344c78a45,
friends: 5f3aa3a9c6140e3344c78a44,
name: 'John',
__v: 0,
friends_places:
{ _id: 5f3aa3a9c6140e3344c78a48,
latitude: 'Place2',
creator: 5f3aa3a9c6140e3344c78a44,
__v: 0 },
friend_name:
{ _id: 5f3aa3a9c6140e3344c78a44,
friends: [Array],
name: 'Natalie',
__v: 0 } },
{ _id: 5f3aa3a9c6140e3344c78a45,
friends: 5f3aa3a9c6140e3344c78a43,
name: 'John',
__v: 0,
friends_places:
{ _id: 5f3aa3a9c6140e3344c78a46,
latitude: 'Place1',
creator: 5f3aa3a9c6140e3344c78a43,
__v: 0 },
friend_name:
{ _id: 5f3aa3a9c6140e3344c78a43,
friends: [],
name: 'Bob',
__v: 0 } },
{ _id: 5f3aa3a9c6140e3344c78a45,
friends: 5f3aa3a9c6140e3344c78a43,
name: 'John',
__v: 0,
friends_places:
{ _id: 5f3aa3a9c6140e3344c78a47,
latitude: 'Place3',
creator: 5f3aa3a9c6140e3344c78a43,
__v: 0 },
friend_name:
{ _id: 5f3aa3a9c6140e3344c78a43,
friends: [],
name: 'Bob',
__v: 0 } } ]
So, we've got a flat list of John's friends' places.
Important bit: from: places in $lookup is critical, as we have to use MongoDB's name of the collection, not model name 1.

Mongoose $lookup object returns empty array

I have tried other similar kind of questions available but nothing seems to work for me.
I have two collections:
leads:
const mongoose = require("mongoose");
const id = mongoose.Schema.Types.ObjectId;
const leadsSchema = mongoose.Schema(
{
_id: id,
userId: { type: id, ref: "User", required: true },
leadName: String,
leads: [
{
_id: id,
name: String,
status: { type: String, required: false, default: "New" },
leadActivity: { type: String, required: false, default: "No Campaign Set" },
headline: { type: String, required: false },
location: { type: String, required: false },
leadType: { type: id, ref: "LeadsCategory", required: true },
}
],
campaignAssociated: {type: id, ref: "campaign"},
},
{
timestamps: true
}
);
module.exports = mongoose.model("lead", leadsSchema);
leadCategory
const mongoose = require("mongoose");
const leadsCategorySchema = mongoose.Schema(
{
_id: mongoose.Schema.Types.ObjectId,
name: {
type: String,
required: false,
},
leadsData: [{ type: Array, ref: "lead" }],
},
{ timestamps: true }
);
module.exports = mongoose.model("LeadsCategory", leadsCategorySchema);
I am trying to reference/populate the name of the lead from leadscategory schema into the leads
exports.get_single_lead_info = (req, res) => {
const { userId } = req.user;
const { leadid } = req.body;
let idToSearch = mongoose.Types.ObjectId(leadid);
Lead.aggregate([
{
$lookup: {from: 'leadscategories', localField: 'leadType', foreignField: 'name', as: 'type as'}
},
{
$match: {
userId: mongoose.Types.ObjectId(userId),
},
},
{
$unwind: "$leads",
},
{
$match: {
"leads._id": idToSearch,
},
},
])
.exec(function (err, result) {
if (err) {
return res.status(400).json({ message: "Unable to fetch data", err });
}
if (!result.length) {
res.status(404).json("No result found");
} else {
res.status(200).json({ message: "Lead info found", result });
}
});
};
But it outputs me the lookup result as an empty array everytime:
{
"message": "Lead info found",
"result": [
{
"_id": "5ece11cbac50c434dc4b7f2c",
"leadName": "python",
"leads": {
"status": "New",
"leadActivity": "Campaign Set",
"name": "Hailey",
"headline": "Machine Learning | Python",
"location": "New Delhi Area, India",
"_id": "5ece11cbac50c434dc4b7f29",
"leadType": "5ebce0f81947df2fd4eb1060"
},
"userId": "5eba83d37d4f5533581a7d58",
"createdAt": "2020-05-27T07:07:55.231Z",
"updatedAt": "2020-05-27T10:47:42.098Z",
"__v": 0,
"type as": [] //<--- Need lead type name associated inside this
}
]
}
Input: "leadid": "5ece11cbac50c434dc4b7f29"
Any help appreciated.
[
{
$match: {
userId: mongoose.Types.ObjectId(userId),
},
},
{
$unwind: "$leads",
},
{
$match: {
'leads._id': idToSearch,
},
},
{
$lookup: {
from: 'leadscategories',
localField: 'leads.leadType',
foreignField: '_id',
as: 'type as'
}
},
]

Resources