Mongoose: Populate a tree with parent referencing top-down - node.js

As I have stated in another question, I am working on a project involving a tree.
The tree uses parent referencing, so every node has the id of its parent
I need to load the tree top-down (from root to children) from the db and replace the parent references by children arrays (because the client needs them)
I've chosen this method, because I estimate 98% of the operation to be create/update on nodes (and this way I only have to create 1 node on update, instead of also updating the parent to add the child to the array) and only about 2% to be read operations (I only have to read the complete tree, there is no use case for reading parts or subtrees)
The Tree models is:
const mongoose = require("mongoose");
const Promise = require("bluebird");
mongoose.Promise = Promise;
const Node = require("./node-model");
const TreeSchema = new mongoose.Schema({
root: { type: Schema.Types.ObjectId, ref: 'Node' },
});
And the Node model:
const mongoose = require("mongoose");
const Promise = require("bluebird");
mongoose.Promise = Promise;
const NodeSchema = new mongoose.Schema({
parent: Schema.Types.ObjectId,
children: [], // to be populated on loading the tree
data: {
d1: String,
//...
}
});
NodeSchema.methods.populateTree = function() {
return this.constructor.find({ parent: this._id }).exec()
.then(function(arrayOfChildren) {
return Promise.each(arrayOfChildren, function(child){
this.children.push(child); // PROBLEM: 'this' is undfined here!
delete child.parent; // delete parent reference because JSON has problems with circular references
return child.populateTree();
});
});
}
Also, there is a tree container:
const TreeContainerSchema = new mongoose.Schema({
owner: { type: Schema.Types.ObjectId, ref: 'User', required: true },
tree: { type: Schema.Types.ObjectId, ref: 'Tree' },
});
I'm trying to load the complete tree (in his container) to send it back to the client as JSON as follows:
getTreeContainerById = function(req, res) {
var promise = TreeContainer.
findById(req.params.id).
populate("owner", "name"). // only include name
populate({
path: "tree",
populate: {
path: "root",
populate: "data"
}
}).exec();
promise.then(function(treeContainer){
return treeContainer.tree.root.populateTree()
.then(function(){ return treeContainer });
}).then(function(treeContainer) {
// I need the tree container here to send it back to the client
res.json(treeContainer);
});
};
But this implementation isn't working. The problems I face are:
In the populateTree schema method, I can't access the current node through "this" (it is undefined) but I need the reference somehow to add the children to the array
If I try child.parent.children.push instead, this also isn't working, because I only have the id of the parent (in child.parent) and not the entity (and I don't think it is the correct approach to load it again from the database)
In an earlier version, I had the problem, that the JSON was send back to the client, before the tree was completely populated, but I think i solved this through the use of the schema method
In general, I don't know, if this is the correct approach to solve my problem (populate the children references and delete the parent references in my tree) or if there is a more appropriate solution
I hope, I could make my problem clear. Any help is much appreciated!

With populateTree as follows it works:
NodeSchema.methods.populateTree = function() {
var node = this;
return this.constructor.find({ parent: this._id }).exec()
.then(function(arrayOfChildren) {
return Promise.each(arrayOfChildren, function(child){
node.children.push(child);
child.parent = null;
return child.populateTree();
});
});
}
Thanks to #danh who suggested the same!

Related

Mongoose | populate on post middleware for "save"

Please consider the following parentSchema and childSchema:
const parentSchema = new mongoose.Schema({
name: String,
children: [{ type: mongoose.Schema.Types.ObjectId, ref: "Child" }],
});
const childSchema = new mongoose.Schema({
name: String,
parent: { type: mongoose.Schema.Types.ObjectId, ref: "Parent" },
});
How do I access the name of the parent within a post middleware of the childSchema?
I am trying the code below but parent is assigned the ObjectId instead of the actual parent model.
childSchema.post("save", async function (child) {
const parent = child.populate("parent").parent;
});
What's the right way to do this?
If you want to populate something after the initial fetch you need to call execPopulate -- e.g.:
childSchema.post("save", async function (child) {
try {
if (!child.populated('parent')) {
await child.populate('parent').execPopulate();
}
const parent = child.populate("parent").parent;
} catch (err) {}
});
I just found out about this.model("Model") so I'm sharing another solution. Not sure how it compares with the accepted one, though:
childSchema.post("save", async function (child) {
try {
const parent = await this.model("Parent").findOne({ _id: child.parent });
parent.children.push(child._id);
parent.save();
} catch (error) {}
});

Include non related collection in model result

I'm new to MongoDb and Mongoose and this might sound silly but I'm a bit confused about how things work.
I have two unrelated models: page model and team model that looks something like this:
// page.js
const mongoose = require('mongoose');
const schema = new mongoose.schema({
name: String,
body: {
title: String,
},
});
const Page = mongoose.model('Page', schema);
export default Page;
and
// team.js
const mongoose = require('mongoose');
const schema = new mongoose.schema({
name: String,
position: Number,
});
const Team = mongoose.model('Team', schema);
export default Team;
What I want to do is when I find one page (Page.findOne({...})) to include all teams. The result will look like this:
{
_id: 'some_id',
name: 'some name',
body: {
title: 'A title',
teams: [
{ name: 'Team1', position: 1 },
{ name: 'Team2', position: 2 },
// ...
{ name: 'Team3', position: 3 },
],
},
}
I looked at populate but this requires refs to other model.
Looked at virtuals but from what I understand this should work only with instance properties.
What will be the best approach to achieve this without adding relations between the two models?
This is I'm currently doing:
const pageResult = await Page.findOne({});
let page = pageResult.toObject();
page.body.team = await Team.find({});
well with out referencing, the only way to do that is to manually query Page model and findOne() what is the doc you want and then inside the callback of that findOne(), you will have to get Teams you desire with the value of Page.body.title value.
But its very easy to use Ref and populate using mongoose but if this is the way you really want to go knock yourself out mate ... :) cheers ...
Page.findOne({_id:req.body.id},(err,page)=>{
if(!err){
team.find({},(err,teams)=>{ // this will give you an array of teams
if(!err){
page.body.teams = teams; // this line set teams array from this callback to previous findOne()'s page obj
//so that you will finally create the object you want
}else{
throw err;
}
});
}else{
}
});
since you have only 2 fields in team model I think you won't be needing to use projections

Sails/Waterline: How to retrieve relations inside a relation?

I need to retrieve an object and also get the relations and nested relations.
So, I have the three models below:
User model:
module.exports = {
attributes: {
name: {
type: 'string'
},
pets: {
collection: 'pet',
via: 'owner',
}
}
Pet model:
module.exports = {
attributes: {
name: {
type: 'string'
},
owner: {
model: 'user'
},
vaccines: {
collection: 'vaccine',
via: 'pet',
}
}
Vaccine model:
module.exports = {
attributes: {
name: {
type: 'string'
},
pet: {
model: 'pet'
}
}
Calling User.findOne(name: 'everton').populate('pets').exec(....) I get the user and associated Pets. How can I also get the associated vaccines with each pet? I didn't find references about this in the official documentation.
I've ran into this issue as well, and as far as I know, nested association queries are not built into sails yet (as of this post).
You can use promises to handle the nested population for you, but this can get rather hairy if you are populating many levels.
Something like:
User.findOne(name: 'everton')
.populate('pets')
.then(function(user) {
user.pets.forEach(function (pet) {
//load pet's vaccines
});
});
This has been a widely discussed topic on sails.js and there's actually an open pull request that adds the majority of this feature. Check out https://github.com/balderdashy/waterline/pull/1052
While the answer of Kevin Le is correct it can get a little messy, because you're executing async functions inside a loop. Of course it works, but let's say you want to return the user with all pets and vaccines once it's finished - how do you do that?
There are several ways to solve this problem. One is to use the async library which offers a bunch of util functions to work with async code. The library is already included in sails and you can use it globally by default.
User.findOneByName('TestUser')
.populate('pets')
.then(function (user) {
var pets = user.pets;
// async.each() will perform a for each loop and execute
// a fallback after the last iteration is finished
async.each(pets, function (pet, cb) {
Vaccine.find({pet: pet.id})
.then(function(vaccines){
// I didn't find a way to reuse the attribute name
pet.connectedVaccines = vaccines;
cb();
})
}, function(){
// this callback will be executed once all vaccines are received
return res.json(user);
});
});
There is an alternative approach solving this issue with bluebird promises, which are also part of sails. It's probably more performant than the previous one, because it fetches all vaccines with just one database request. On the other hand it's harder to read...
User.findOneByName('TestUser')
.populate('pets')
.then(function (user) {
var pets = user.pets,
petsIds = [];
// to avoid looping over the async function
// all pet ids get collected...
pets.forEach(function(pet){
petsIds.push(pet.id);
});
// ... to get all vaccines with one db call
var vaccines = Vaccine.find({pet: petsIds})
.then(function(vaccines){
return vaccines;
});
// with bluebird this array...
return [user, vaccines];
})
//... will be passed here as soon as the vaccines are finished loading
.spread(function(user, vaccines){
// for the same output as before the vaccines get attached to
// the according pet object
user.pets.forEach(function(pet){
// as seen above the attribute name can't get used
// to store the data
pet.connectedVaccines = vaccines.filter(function(vaccine){
return vaccine.pet == pet.id;
});
});
// then the user with all nested data can get returned
return res.json(user);
});

mongoose recursive populate

I have been searching for a while and I didn't find any good answer. I have n-deep tree that I am storing in DB and I would like to populate all parents so in the end I get the full tree
node
-parent
-parent
.
.
-parent
So far I populate to level 2, and as I mentioned I need to get to level n.
Node.find().populate('parent').exec(function (err, items) {
if (!err) {
Node.populate(items, {path: 'parent.parent'}, function (err, data) {
return res.send(data);
});
} else {
res.statusCode = code;
return res.send(err.message);
}
});
you can do this now (with https://www.mongodb.com/blog/post/introducing-version-40-mongoose-nodejs-odm)
var mongoose = require('mongoose');
// mongoose.Promise = require('bluebird'); // it should work with native Promise
mongoose.connect('mongodb://......');
var NodeSchema = new mongoose.Schema({
children: [{type: mongoose.Schema.Types.ObjectId, ref: 'Node'}],
name: String
});
var autoPopulateChildren = function(next) {
this.populate('children');
next();
};
NodeSchema
.pre('findOne', autoPopulateChildren)
.pre('find', autoPopulateChildren)
var Node = mongoose.model('Node', NodeSchema)
var root=new Node({name:'1'})
var header=new Node({name:'2'})
var main=new Node({name:'3'})
var foo=new Node({name:'foo'})
var bar=new Node({name:'bar'})
root.children=[header, main]
main.children=[foo, bar]
Node.remove({})
.then(Promise.all([foo, bar, header, main, root].map(p=>p.save())))
.then(_=>Node.findOne({name:'1'}))
.then(r=>console.log(r.children[1].children[0].name)) // foo
simple alternative, without Mongoose:
function upsert(coll, o){ // takes object returns ids inserted
if (o.children){
return Promise.all(o.children.map(i=>upsert(coll,i)))
.then(children=>Object.assign(o, {children})) // replace the objects children by their mongo ids
.then(o=>coll.insertOne(o))
.then(r=>r.insertedId);
} else {
return coll.insertOne(o)
.then(r=>r.insertedId);
}
}
var root = {
name: '1',
children: [
{
name: '2'
},
{
name: '3',
children: [
{
name: 'foo'
},
{
name: 'bar'
}
]
}
]
}
upsert(mycoll, root)
const populateChildren = (coll, _id) => // takes a collection and a document id and returns this document fully nested with its children
coll.findOne({_id})
.then(function(o){
if (!o.children) return o;
return Promise.all(o.children.map(i=>populateChildren(coll,i)))
.then(children=>Object.assign(o, {children}))
});
const populateParents = (coll, _id) => // takes a collection and a document id and returns this document fully nested with its parents, that's more what OP wanted
coll.findOne({_id})
.then(function(o){
if (!o.parent) return o;
return populateParents(coll, o.parent))) // o.parent should be an id
.then(parent => Object.assign(o, {parent})) // replace that id with the document
});
Another approach is to take advantage of the fact that Model.populate() returns a promise, and that you can fulfill a promise with another promise.
You can recursively populate the node in question via:
Node.findOne({ "_id": req.params.id }, function(err, node) {
populateParents(node).then(function(){
// Do something with node
});
});
populateParents could look like the following:
var Promise = require('bluebird');
function populateParents(node) {
return Node.populate(node, { path: "parent" }).then(function(node) {
return node.parent ? populateParents(node.parent) : Promise.fulfill(node);
});
}
It's not the most performant approach, but if your N is small this would work.
Now with Mongoose 4 this can be done. Now you can recurse deeper than a single level.
Example
User.findOne({ userId: userId })
.populate({
path: 'enrollments.course',
populate: {
path: 'playlists',
model: 'Playlist',
populate: {
path: 'videos',
model: 'Video'
}
}
})
.populate('degrees')
.exec()
You can find the official documentation for Mongoose Deep Populate from here.
Just don't :)
There is no good way to do that. Even if you do some map-reduce, it will have terrible performance and problems with sharding if you have it or will ever need it.
Mongo as NoSQL database is really great for storing tree documents. You can store whole tree and then use map-reduce to get some particular leafs from it if you don't have a lot of "find particular leaf" queries. If this doesn't work for you, go with two collections:
Simplified tree structure: {_id: "tree1", tree: {1: [2, {3: [4, {5: 6}, 7]}]}}. Numbers are just IDs of nodes. This way you'll get whole document in one query. Then you just extract all ids and run second query.
Nodes: {_id: 1, data: "something"}, {_id: 2, data: "something else"}.
Then you can write simple recurring function which will replace node ids from first collection with data from second. 2 queries and simple client-side processing.
Small update:
You can extend second collection to be a little more flexible:
{_id: 2, data: "something", children:[3, 7], parents: [1, 12, 13]}
This way you'll be able to start your search from any leaf. And then, use map-reduce to get to the top or to the bottom of this part of tree.
This is a more straight forward approach to caub's answer and great solution. I found it a bit hard to make sense of at first so I put this version together.
Important, you need both 'findOne' and 'find' middleware hooks in place for this solution to work. *
* Also, the model definition must come after the middleware definition *
const mongoose = require('mongoose');
const NodeSchema = new mongoose.Schema({
children: [mongoose.Schema.Types.ObjectId],
name: String
});
const autoPopulateChildren = function (next) {
this.populate('children');
next();
};
NodeSchema
.pre('findOne', autoPopulateChildren)
.pre('find', autoPopulateChildren)
const Node = mongoose.model('Node', NodeSchema)
const root = new Node({ name: '1' })
const main = new Node({ name: '3' })
const foo = new Node({ name: 'foo' })
root.children = [main]
main.children = [foo]
mongoose.connect('mongodb://localhost:27017/try', { useNewUrlParser: true }, async () => {
await Node.remove({});
await foo.save();
await main.save();
await root.save();
const result = await Node.findOne({ name: '1' });
console.log(result.children[0].children[0].name);
});
I tried #fzembow's solution but it seemed to return the object from the deepest populated path. In my case I needed to recursively populate an object, but then return the very same object. I did it like that:
// Schema definition
const NodeSchema = new Schema({
name: { type: String, unique: true, required: true },
parent: { type: Schema.Types.ObjectId, ref: 'Node' },
});
const Node = mongoose.model('Node', NodeSchema);
// method
const Promise = require('bluebird');
const recursivelyPopulatePath = (entry, path) => {
if (entry[path]) {
return Node.findById(entry[path])
.then((foundPath) => {
return recursivelyPopulatePath(foundPath, path)
.then((populatedFoundPath) => {
entry[path] = populatedFoundPath;
return Promise.resolve(entry);
});
});
}
return Promise.resolve(entry);
};
//sample usage
Node.findOne({ name: 'someName' })
.then((category) => {
if (category) {
recursivelyPopulatePath(category, 'parent')
.then((populatedNode) => {
// ^^^^^^^^^^^^^^^^^ here is your object but populated recursively
});
} else {
...
}
})
Beware it's not very efficient. If you need to run such query often or at deep levels, then you should rethink your design
Maybe a lot late for that but mongoose has some documentation on this :
Ancestors Tree Array
Materialized Path Tree Array
I think the first one is more appropriate to you as you are looking to populate parents.
With that solution, you can with one regex query, search all the documents matching your designered output tree.
You would setup documents with this Schema :
Tree: {
name: String,
path: String
}
Paths field would be the absolute path in your tree :
/mens
/mens/shoes
/mens/shoes/boots
/womens
/womens/shoes
/womens/shoes/boots
For example you could search all the childrens of your node '/mens/shoes' with one query :
await Tree.find({ path: /^\/mens/shoes })
It would return all the documents where the path starts with /mens/shoes :
/mens/shoes
/mens/shoes/boots
Then you'd only need some client-side logic to arrange it in a tree structure (a map-reduce)

Only the first sub-document added to MongoDB model

I have a Mongoose model that looks like this:
var ProjectSchema = new Schema({
name: String,
slug: String,
dateCreated: { type: Date, default: new Date() },
dateUpdated: { type: Date, default: new Date() },
createdByUserId: Schema.Types.ObjectId,
screens: [Schema.Types.Mixed]
});
I have a class method that looks like this:
ProjectSchema.statics.saveElementProperties = function(slugName, screenIndex, elementId, props, callback) {
var Project = mongoose.model('Project');
var updateProject = function(project) {
// Init empty objects if missing
project.screens[screenIndex] = project.screens[screenIndex] || {};
project.screens[screenIndex].elements = project.screens[screenIndex].elements || {};
project.screens[screenIndex].elements[elementId] = project.screens[screenIndex].elements[elementId] || {};
// Apply properties
project.screens[screenIndex].elements[elementId] = "Dummy Project Data";
console.log('elements before save:', project.screens[screenIndex].elements);
project.save(callback);
};
Project.findOne({ slug: slugName }, function(err, project) {
if (!project) {
project = new Project({ name: slugName, slug: slugName });
}
updateProject(project);
});
};
This happens when I call the method saveElementProperties:
The first time I run this method, it works like it should; a new object is added to project.screens[screenIndex].elements both in runtime (the 'elements before save:' log statement) and when checking the MongoDB database with the mongo client.
The second time, a 2nd object is added to project.screens[screenIndex].elements in runtime, but this object is never persisted to MongoDB.
The third time, object 1 and 3 are visible in project.screens[screenIndex].elements in runtime, but the 3rd object is never persisted to MongoDB.
What causes this behavior?
MAJOR UPDATE: I rewrote the entire persistence mechanism to run less frequently, and instead replace the entire project.screens[screenIndex].elements object with an updated structure:
ProjectSchema.statics.saveScreenProperties = function(slugName, screenIndex, elements, callback) {
console.log('saveScreenProperties:', slugName, screenIndex);
var Project = mongoose.model('Project');
var updateProject = function(project) {
// Init empty objects if missing
project.screens[screenIndex] = project.screens[screenIndex] || {};
project.screens[screenIndex].elements = elements;
// Mark as modified and save
project.markModified('screens.' + screenIndex);
project.save(callback);
};
Project.findOne({ slug: slugName }, function(err, project) {
if (!project) {
project = new Project({ name: slugName, slug: slugName });
console.log(' creating new project');
}
updateProject(project);
});
};
However, it still shows the same behavior - it stores the initial elements object, but not later changes to it.
The problem is that you are manipulating the objects just as you would in standard code but mongoose is not a 'persistence engine` it's an ODM. As such you need to use the provided methods to update the datastore.
You will need to use the update operators to $push onto the array. Mongoose has some analogous methods as well. The most explainatory docs are on the MongoDB site.
http://docs.mongodb.org/manual/reference/operator/update-array/
http://docs.mongodb.org/manual/reference/operator/update/
http://docs.mongodb.org/manual/reference/method/db.collection.update/#db.collection.update

Resources