Fetch data from multiple collections in mongo - node.js

I have some collections shown like below which is holding relationships, relation between testMaster and testDoc is holding inside the testDocMaster
For eg:
testMaster {
_id: "Schema.Objectid",
name: "String" //master name
}
testDoc {
_id : Schema.ObjectId,
name: "String", //doc name
other datas
}
testDocMaster {
masterId: "_id of the testMaster table",
docId : "_id of the above testDoc"
}
For each master entry, we are expecting many relations,
what would be the best way to fetch the data from the testDoc table, if I have the masterId.

I got it working using this:
// GLOBAL ARRAYS FOR STORING COLLECTION DATA
var collectionOne = [];
var collectionTwo = [];
app.get('/', function(req, res){
MongoClient.connect("mongodb://localhost:27017/michael", function(err, db) {
if(!err) {
console.log("We are connected");
}
db.collection("collectionOne", function(err, collection) {
collection.find().sort({order_num: 1}).toArray(function(err, result) {
if (err) {
throw err;
} else {
for (i=0; i<result.length; i++) {
collectionOne[i] = result[i];
}
}
});
db.collection("collectionTwo", function(err, collection) {
collection.find().sort({order_num: 1}).toArray(function(err, result) {
if (err) {
throw err;
} else {
for (i=0; i<result.length; i++) {
collectionTwo[i] = result[i];
}
}
});
});
// Thank you aesede!
res.render('index.html', {
collectionOne: collectionOne,
collectionTwo: collectionTwo
});
});
});
});
Now, for some reason when Node restarts, and I hit refresh, it doesn't render HTML into the front-end. However, any subsequent refresh renders the page correctly.

Assuming your testDocMaster schema uses ObjectId types that ref the other two collections you can use Mongoose's query population support to help with this:
TestDocMaster.findOne({ masterId: masterId})
.populate('docId')
.exec(function(err, testDocMaster) {
// testDocMaster.docId is populated with the full testDoc for the
// matching _id
});

Related

How to get data from one collection and insert into another collection in Nodejs?

Am using Nodejs and MongoDB and I am new to nodejs. I need to know how to get data from one collection and append some additional data and insert into another collection.
db.collection('collection1').find({ "Id" : 12345 }).toArray(function(err, result){
db.collection('collection2', function(err, collection){
collection.insert({
//some data
})
})
})
When I try this code its not working its giving me error insert is not defined.
thanks,
John.
db.collection('collection1').find({ "Id" : 12345 }).toArray(function(err, result){
//do the modification here
db.collection('collection2').insert(modifiedResult, function(err, result){
if(err) {
//log error
}else{
//log result
}
})
})
One more thing, If the result array length is more that one and you want to insert then separately then use promise
db.collection('collection1').find({ "Id" : 12345 }).toArray(function(err, result){
//do the modification here
Promise.all(modifiedResult.map((eachModifiedResult)=>{
return db.collection('collection2').insert(eachModifiedResult);
}).then((result)=>{
//result of the insert
}).catch((err){
//err if any happen
});
})
But if you have a very large doc then do it as Neil Said. Read the collection one by one using cursor and modify them and insert them to other db.
You can use callback library like async or Promises Q
Promise
var collectionData = null;
var modifiedResult = null;
// here i am using async library to avoid callbackHell
async.series([
// for get data from collection 1.
function(cb) {
var criteria = {
"Id": 12345
}
db.collection('collection1').find(criteria).toArray(function(dbErr, dbResult) {
if (err) {
cb(dbErr)
} else {
collectionData = dbResult;
cb()
}
})
},
// Append Data in collectionData
function(cb) {
// do you work here to append data in collectionData
modifiedResult = extendedData; // this is just an example you need to work on it
cb();
},
// Update collection 2 here
function(cb) {
db.collection('collection2').insert(modifiedResult, function(err, result) {
if (err) {
cb(dbErr)
} else {
collectionData = dbResult;
cb()
}
});
}
]);

mongodb native driver error on query

I am writing the filter using mongodb native driver, but it's driving me this error when you run the query.
In the case of this driver, it has no exec?
What is another way to perform this query?
exports.findAll = function(req, res) {
MongoClient.connect(url, function(err, db) {
var section = req.params.section;
var collection = db.collection(section);
var filter = req.query.filter ? {nameToLower: new RegExp('^' + req.query.filter.toLowerCase())} : {};
var query = collection.find(filter);
var count = 0;
collection.count(filter, function (error, result) {
count = result;
});
if(req.query.order) {
query.sort(req.query.order);
}
if(req.query.limit) {
query.limit(req.query.limit);
if(req.query.page) {
query.skip(req.query.limit * --req.query.page);
}
}
query.exec(function (error, results) {
res.json({
count: count,
data: results
});
});
});
};
Error:
TypeError: undefined is not a function
Better to use the async library in this case as it simplifies the code. In the case where you need to run multiple tasks that depend on each other and when they all finish do something else, use the
async.series() module. The following demonstrates how you can go about this in your case:
exports.findAll = function(req, res) {
var locals = {},
section = req.params.section,
filter = !!req.query.filter ? {nameToLower: new RegExp('^' + req.query.filter.toLowerCase())} : {};
async.series([
// Connect to DB
function(callback) {
MongoClient.connect(url, function(err, db) {
if (err) return callback(err);
locals.collection = db.collection(section); //Set the collection here, so the next task can access it
callback();
});
},
// Get count
function(callback) {
locals.collection.count(filter, function (err, result){
if (err) return callback(err);
locals.count = result; //Set the count here
callback();
});
},
// Query collection
function(callback) {
var cursor = locals.collection.find(filter);
if(req.query.order) {
cursor = cursor.sort(req.query.order);
}
if(req.query.limit) {
cursor = cursor.limit(req.query.limit);
if(req.query.page) {
cursor = cursor.skip(req.query.limit * --req.query.page);
}
}
cursor.toArray(function(err, docs) {
if (err) return callback(err);
locals.docs = docs;
callback();
});
}
], function(err) { //This function gets called after the three tasks have called their "task callbacks"
if (err) return next(err);
// Here locals will be populated with 'count' and 'docs'
res.json({
count: locals.count,
data: locals.docs
});
res.render('user-profile', locals);
});
};

Why can not read property when find data in node with mongoDB?

Below is my code
var mongodb = require('mongodb');
var MongodbClient = mongodb.MongoClient;
MongodbClient.connect('mongodb://localhost/test', function(err, db) {
db.collection('contact', function(err, collection) {
collection.find({}, function(err, rows) {
for(var index in rows)
console.log(rows[index]);
});
});
var contact = db.collection('contact');
contact.insert({
name:'Fred',
tel:'123456789',
address: 'Mars',
}, function(err, docs) {
if(err){
console.log("failed")
return;
}
else{
console.log('Success');
}
});
contact.find({}, function(err, docs) {
if(err) {
console.log("Can not find any!");
return;
}
for(var index in docs) {
console.log(docs.length);
var doc = docs[index];
console.log(doc.name);
}
});
});
I can find the data using mongodb shell but in node, it shows
TypeError: Cannot read property 'name' of null
on console.log(doc.name).
and also shows "undefined" when try to console.log(docs.length);
Did I do something wrong?
You are not using correctly the mongodb node API:
You should have:
contact.find({}).toArray(function(err,results) {
if (err) {
console.debug(err);
return;
}
console.debug(JSON.stringify(results));
});
More on docs.
You can try JSON.stringify(object) and then JSON.parse(object) and then try to access the property.
let xxxxxxxx = async (req, res) => {
let resultSet = [];
let object;
let perDayData = await yyyyyyyyyyy.find({});
perDayData = JSON.stringify(perDayData);
perDayData = JSON.parse(perDayData);
console.log(perDayData[0].field);
}

Not able to access result returned from mongoDb outside query

I am working with node.js using express and mongoDb.
I am trying to retrieve data from database "local" and collection "borrower" and pushing it into an empty array collectionOne. It is rendering this array on the view page.
I want to use _id pushed in an array b_ids from this output to make a call to company_details collection under same router .
The issue I am facing here is I am not getting any value in array b_ids when trying to access it under call to company_details collection.
The code is as below:
var collectionOne = [];
var collectionTwo = [];
var b_ids = [];
router.get('/fetch',function(req, res, next){
MongoClient.connect('mongodb://localhost:27017/local', function(err, db){
var collection = db.collection('borrower');
db.collection("borrower", function(err, collection) {
collection.find().toArray(function(err, result) {
if (err) {
throw err;
} else {
//console.log(result[0].first_name);
for (i=0; i<result.length; i++) {
collectionOne[i] = result[i];
b_ids[i] = result[i]._id;
}
}
});
db.collection("company_details", function(err, collection){
console.log(b_ids);
collection.find({borrower_id : {$in :b_ids}}).toArray(function(err, result){
if (err) {
throw err;
} else {
for (i=0; i<result.length; i++) {
collectionTwo[i] = result[i];
}
}
});
});
res.render('index',{data : collectionOne , data1 : collectionTwo});
});
});
});
Please suggest me how can I access the b_ids array here. Thanks in advance :)
The callback from toArray is asynchronous, i.e. will happen after your console.log line executes, therefore you need to re-arrange as follows:
var collectionOne = [];
router.get('/fetch',function(req, res, next){
MongoClient.connect('mongodb://localhost:27017/local', function(err, db){
var collection = db.collection('borrower');
db.collection("borrower", function(err, collection) {
collection.find().toArray(function(err, result) {
if (err) {
throw err;
} else {
for (i=0; i<result.length; i++) {
collectionOne[i] = result[i];
b_ids[i] = result[i]._id;
}
db.collection("company_details", function(err, collection){
console.log(b_ids);
collection.find({borrower_id : {$in :b_ids}}).toArray(function(err, result){
if (err) {
throw err;
} else {
for (i=0; i<result.length; i++) {
collectionTwo[i] = result[i];
}
res.render('index',{data : collectionOne , data1 : collectionTwo});
}
});
});
}
});
});
});
This ensures that the callback completes before you try and log / render it.
It is simple create a function with a callback function inside it as an argument
function returnBids(collection, callback){
collection.find().toArray(function(err, result) {
if (err)
throw err;
else
callback(result)
})
}
and you can access your result outside by calling the function returnbids() .i.e
returnBids(collection,function(b_ids){
console.log(b_ids)
})

Memory leak in migrating to new mongoose collection from existing collection

I have a multiple unnecessarily large, and poorly unorganized collections in mongodb and I am trying to migrate a subset of that data to a few new collections that have a mongoose schema. However, my approach seems to have a memory leak, as the migration slowly grinds to a halt and the node process eats up all the RAM after a while. Resulting in this error:
FATAL ERROR: JS Allocation failed - process out of memory
I start off with a list of the documents I want to migrate over, people. Initially, I used a forEach over people, but that resulted in a very big memory leak with many cursors being created before being closed. I then tried using async to generate each query in series, which is much better than my initial attempt, but there still seems to a slight memory leak, and eventually the error occurs after only a small part of the data has been migrated. What am I missing?
var PersonSchema = new mongoose.Schema({
_id: String,
records: [{
id: String
date: Date
}]
});
var Person = mongoose.model('Person', PersonSchema);
db = mongoose.connection.db;
// Collections I want to use to use in populating new schema
var members = db.collection('members');
var records = db.collection('records');
// Populate members
members.find().each(function (err, p) {
if (err) {
winston.error(err);
}
if (p) {
var id = p['Member ID - Consist'];
var person = new Patient({
_id: id,
records: []
});
person.save(function (err) {
if (err) {
winston.error(err);
} else {
winston.info('Saved %s', id);
}
});
} else {
winston.info('Done importing members');
}
});
// Loop through people again and add all their records
Person.find().exec(function (err, people) {
async.eachSeries(people, function (p, callback) {
var cursor = records.find({
'memberId': p.id
});
cursor.each(function (err, doc) {
if (err) {
throw new Error(err);
}
if (doc) {
p.records.push({
id: doc['recordId'],
date: new Date(doc['date'])
});
} else {
p.save(function (err) {
if (err) {
winston.error(err);
} else {
cursor.close(function () {
callback();
});
winston.info('Data saved for %s', p.id);
}
});
}
});
}, function (err) {
console.log(err);
});
});

Resources