Generate JSON tree from mongoDB data in node.js - node.js

There are lot of questions like this, but unfortunately I dont find any of them fitting my case.
Closest is Create a JSON tree in Node.Js from MongoDB but it still doesnt work as expected.
Or maybe my head cant wrap this problem...
I have schema that key components for my problem looks like this:
var userSchema = new Schema({
_id: {type: Number},
children: [{type: Number, ref: 'User'}]
)};
each user may have three children users, so it can go infinately deep.
Fortunately, i have to cover two scenarios -
build json tree from specific user up to 3 nestings
calculate data for 10 nestings from specific root.
I tried to write recursive function like this in my express.js api:
api.get('/user/tree/:user_id', function (req, res) {
var user_id = req.params.user_id;
var depth = 0;
var root = {};
function walker(parent) {
if (depth >= 3) {
return res.send('whole data, not just last user'); // this is wrong. it will try to res.send for each iteration of forEach, and it sends only last user.
}
depth += 1;
_.forEach(parent.mlm.childs, function (userid, index) {
User.findOneAsync({_id: userid}).then(function(user) {
parent.mlm.childs[index] = user;
walker(parent.mlm.childs[index]);
});
});
}
User.findOneAsync({_id: user_id}).then(function(user) {
root = user;
walker(user, root);
});
});
but of course it only traverse the tree, instead of traverse and create whole JSON.
Im stuck on how to be able to access the root and send whole tree.
problem of sending many res.send can be solved by counting iterations and send only if forEach ended, i guess.
Thanks for any help.

Ok. I found a solution.
api.get('/user/tree/:user_id', function (req, res) {
var user_id = req.params.user_id;
var tree = {};
var counter = 0;
var gloCounter = 0;
function walker(parent) {
gloCounter += 1;
if (parent.mlm.childs.length === 0 && gloCounter > counter) {
res.send(tree);
return;
}
_.forEach(parent.mlm.childs, function (child, i) {
counter += 1;
User.findOneAsync({_id: child})
.then(function(child) {
parent.mlm.childs[i] = child;
var newParent = parent.mlm.childs[i];
walker(newParent);
}).catch(function(err) {
console.log('error: ' + err);
});
});
}
User.findOneAsync({_id: user_id})
.then(function(user) {
tree = user;
walker(tree);
}).catch(function(err) {
console.log('err: ' + err);
});
});
It works as expected - traverse through whole structure, and create json, that is sent back.
It uses lodash and bluebird promises, for those who would solve similar problem in future and dont understand what is happening with all those "Async" sufixes and _.forEach.

Related

Assign keystonejs callback function data to array

I'm new to node.js and currently working on a project using keystonejs cms and MongoDB. Now I'm stuck in getting data related to multiple collections. Because of this callback functions, I couldn't return an array with relational data. My code something similar to this sample code.
var getAgenda = function(id, callback){
callback = callback || function(){};
if(id){
AgendaDay.model.find({summit:id}).exec(function (err, results3) {
var arr_agenda = [];
var arr_agenda_item = [];
for(var key3 in results3){
AgendaItem.model.find({agendaDay:results3[key3]._id}).exec(function (err, results2){
for(var key2 in results2){
arr_agenda_item.push(
{
item_id: results2[key2]._id,
item_name: results2[key2].name,
from_time: results2[key2].time_from,
to_time: results2[key2].time_to,
desc: results2[key2].description,
fatured: results2[key2].featured,
}
);
}
arr_agenda.push(
{
name: results3[key3].name,
date: results3[key3].date,
description: results3[key3].description,
item_list:arr_agenda_item
}
);
return callback(arr_agenda);
});
}
});
}
}
exports.list = function (req, res) {
var mainarray = [];
Summit.model.find().exec(function (err, resultssummit) {
if (err) return res.json({ err: err });
if (!resultssummit) return res.json('not found');
Guest.model.find().exec(function (err, resultsguset) {
for(var key in resultssummit){
var agen_arr = [];
for(var i=0; i<resultssummit[key].guests.length; i++){
var sumid = resultssummit[key]._id;
//this is the function im trying get data and assign to mainarray
getAgenda(sumid, function(arr_agenda){
agen_arr = arr_agenda;
});
mainarray.push(
{
id: resultssummit[key]._id,
name: resultssummit[key].name,
agenda_data: agen_arr,
}
);
}
res.json({
summit: mainarray,
});
}
});
}
}
If anyone can help me out, that would be really great :)
You need to restructure this whole thing. You should not be calling mongo queries in a for loop and expecting their output at the end of the loop. Also, your response is in a for loop. That won't work.
I'll tell you how to do it. I cannot refactor all of that code for you.
Instead of putting mongodb queries in a for loop, you need to convert it in a single query. Just put the _ids in a single array and fire a single query.
AgendaItem.model.find({agendaDay:{$in:ARRAY_OF_IDS}})
You need to do the same thing for AgendaDay.model.find({summit:id}) as well.

Nested query Synchronization in mongodb

I have following two collection:
1) Users: {name: xyz,
email:abc#xuz.com}
2) Posts: {_id: 12345678,
time:asdfg,
authEmail:abc#xyz.com,
description: asdigligvilud}
Here i want to get the details of each post along with the corresponding name of author id.
So i wrote query like this:
// get all the posts
Posts.find((err, posts)=> {
if (err) { next(err) };
var data = Array();
var count = 0;
var len = posts.length;
// function to check the end of inner queries
var checkloop = function(){
count++;
if(count===len)
return res.json({result:data,
msg:'success'});
}
for(var i=0;i<len;i++){
Users.findOne({email:posts[count].authEmail},(err,usr)=>{
if(usr){
var item = {
'authorName':usr.name?
'email':usr.email,
'postDesc':posts[count].desc,
'creationTime':posts[count].time
}
data.push(item);
}
checkloop();
});
}
});
But i am getting same result for each iterations that is the post deatils and author id corresponding to the very first loop that is posts[0] only.
So i think its because of asynchronous nature of queries. Is there any standard way to make such query in mongodb or do i need to change my callback method?
Thanks in advance.
I believe the issue you had was because count inside the for loop probably was meant to be i. In addition, each loop iteration captures the same variable i in the Users.findOne callback, but the callback should be able to refer to the post that was queried. Without making much changes, I think this can work with using forEach over the returned posts like below:
// get all the posts
Posts.find((err, posts)=> {
if (err) { next(err) };
var data = Array();
var count = 0;
var len = posts.length;
// function to check the end of inner queries
var checkloop = function(){
count++;
if(count===len)
return res.json({result:data,
msg:'success'});
}
posts.forEach((post) => {
Users.findOne({email:post.authEmail},(err,usr)=>{
if(usr){
var item = {
'authorName':usr.name?
'email':usr.email,
'postDesc':post.desc,
'creationTime':post.time
}
data.push(item);
}
checkloop();
}
});
});

Retrieve data from MongoDB and save it to global object in Node.js and Express.js

I'm trying to get data from MongoDB collection and then save it to a global object.Later I need to parse it to HTML template.
Here is my code:
When user log onto his profile: then we need to get his projects and here we call findeprojects() function
usrRouter.route('/profile')
.all(function (req,res,next) {
if(!req.user){
res.redirect('/');
}
next();
})
.get(function (req,res,userObj) {
// var proj = findprojects();
userObj = req.user;
var pro = {};
pro = findprojects(userObj);
res.render('index',{name:userObj.username, email:userObj.email});
//res.sendFile('profile.html',{root:path.join(__dirname,'../public'),},{name:userObj.username});
});
Here is findeprojects function code:
var findprojects = function(obj) {
var usern = obj.username;
mongodb.connect(url,function(err, db){
if(err) throw err;
var collection = db.collection('projects');
//console.log(usern);
collection.find({'pusername':usern});
cursor =db.collection('projects').find({ 'pusername': usern }).toArray(function(err,items){
//console.log(items);
var i;
for(i=0; i<items.length;){
userProjects.createdBy = items[i].pusername;
userProjects.proName = items[i].projectName;
userProjects.proType = items[i].projectType;
userProjects.proDesc = items[i].projectDesc;
//return userProjects;
i = i+1;
}
});
console.log(userProjects);
});
};
I have declared global object at the top like:
userProjects = {
createdBy:'',
proName:'',
proType:'',
proDesc:''
};
But when I console userprojects object after calling the findeprojects() function it displays empty values.
why dont you use mongoose to model your stuff.
its more intuitive and you no need to declare the global object and do the mapping in the for loop that you are doing.
also your approach is a bit wrong in terms of when you iterate through for aren't you overwriting ?
say you have two documents where pusername is abdul.
so in your case you loose first object which will get overwritten by the second one.
i see that you commented out a return statement but even that wont work properly.
from a design point of view your approach is not efficient.
in mongoose you can do:
{
var userProjectSchema = new mongoose.Schema({
createdBy: { type: String }
, proName: String
, proType: String
, proDesc: String
});
// Find a single document by username.
userProjectSchema.findOne({ pusername : 'abdul' }, function(err, resDoc) {
if (err) return console.error(err);
// do your html stuff here
});
// Find all documents.
userProjectSchema.find(function(err, results) {
if (err) return console.error(err);
// do your html stuff here
});
}

Store values from spookyjs environment into mongoDB

I am trying to scrape data from site by spookyjs and store in mongoDB.I am able to get data from the website.But not able to save scraped data from spookyjs environment to mongoDB.To save scraped data,I passed my database model instance to spookyjs .I refered below link for it.
https://github.com/SpookyJS/SpookyJS/wiki/Introduction
Below is my code where I extracted data in prod_link_info variable and pass its values into mongoDB
var product_model = require('./product').product_model;
//get results
spooky.then([{product_model:product_model},function(){
this.waitForSelector('li[id^="product_"]', function() {
// Get info on all elements matching this CSS selector
var prod_link_info = this.evaluate(function() {
var nodes = document.querySelectorAll('li[id^="product_"]');
return [].map.call(nodes, function(node) { // Alternatively: return Array.prototype.map.call(...
return node.querySelector('a').getAttribute('href')+"\n";
});
});
//insert values in mongodb
for (var i = 0; i < prod_link_info.length; i++) {
product_model.create(
{
prod_link_info:prod_link_info[i],
}, function(err, product){
if(err) console.log(err);
else console.log(product);
});
} });
}]);
Below is the code of database schema and model used in above code.
var mongoose=require('mongoose');
var Schema = mongoose.Schema;
// create a schema
var productSchema = new Schema({
prod_link_info: String,
});
var product_model= mongoose.model('product_model', productSchema);
module.exports = {
product_model: product_model
}
But when I run above code it gives me following error ReferenceError: Can't find variable: product_model.
I want to store the data extracted from spookyjs to mongoDB.Please suggest where am I doing wrong.
When you pass hash of variables to spooky, it is converted to a string using JSON.stringify and then gets converted back to an object using JSON.parse in casper environment (please refer docs); so it is impossible to pass mongoose model to casper environment (moreover there is no actual reason for that).
To solve the problem, you should pass the data from Spooky (casper) environment. As far as I know, the only way to do is to emit data and then handle it using spooky.on. Your example should look like:
var product_model = require('./product').product_model;
//get results
spooky.then([{},function(){
this.waitForSelector('li[id^="product_"]', function() {
// Get info on all elements matching this CSS selector
var prod_link_info = this.evaluate(function() {
var nodes = document.querySelectorAll('li[id^="product_"]');
return [].map.call(nodes, function(node) { // Alternatively: return Array.prototype.map.call(...
return node.querySelector('a').getAttribute('href')+"\n";
});
});
this.emit('data.ready', prod_link_info);
});
}]);
spooky.on('data.ready', function (prod_link_info) {
//insert values in mongodb
for (var i = 0; i < prod_link_info.length; i++) {
product_model.create(
{
prod_link_info:prod_link_info[i],
}, function(err, product){
if(err) console.log(err);
else console.log(product);
});
}
});

Using mongoose middleware to add async virtuals

In a node.js / Mongoose project, I have a schema which contains references to external image files.
var PageSchema = new Schema({
title: String
, media: {
digest: String
, name: String
}
});
Those files have additional properties which are stored in the file itself: url, width, height, exif fields, etc. Those fields will need to be populated before the model being sent to res.render().
For some fields, things are synchronous and a virtual just does the job:
PageSchema.virtual('media.url').get(function () {
return appPaths.fileUrl(this.media);
});
However, width / height, or exif fields require async calls. I thought of using middleware to populate them, but this does not seem to work:
PageSchema.post('init', function(next) {
var media = this.media;
var fileName = filedb.absoluteFilePath(media);
im.identify(fileName, function(err, features) {
if (err) {
media.width = 0;
media.height = 0;
} else {
media.width = features.width;
media.height = features.height;
}
next();
});
});
What am I doing wrong? Is there a common design pattern for solving this kind of problem? (Other than duplicating this information in the database itself?)
The real problem here is that mongoose currently seems to have a wonky implementation of post callbacks. While pre('init',function(next){ ... }); works as you expect, post('init',function(next){ ... }); does not actually get passed a next function. In fact, the post init callback does not receive any arguments whatsoever when it is called.
As such, I usually write a wrapper for my query callbacks to make a sort of DIY middleware:
var setAsyncVirtuals = function(callback){
return function(err, docs){
if(err) return callback(err);
var i = done = docs.length;
if(i > 0)
while(i--){
(function(i){
var filename = getFilename();
im.identify(filename, function(err, features) {
if (err) {
docs[i].media.width = 0;
docs[i].media.height = 0;
} else {
docs[i].media.width = features.width;
docs[i].media.height = features.height;
}
done--;
if(done <= 0) callback(null, docs);
});
})(i); // bind i to hold value for async call
}
else callback(null, docs);
}
}
then
Page.find({}, setAsyncVirtuals(function(err,docs){
res.send(docs); // these have media.width & media.height assigned
}));

Resources