Foreach loop with mongoDB call / node.js - node.js

result.forEach(element => {
//Get each element
console.log("LOOP");
dbo.collection("users").findOne({email: emailGiven, "friends.email": element.email},function(errT, resultT) {
if (errT){
console.log("Query Error Inside!");
res.status(errT.status); // or use err.statusCode instead
console.log(errT);
//db.close();
//return res.send(errT.message);
}
else {
if (resultT) {
var oneUser = {
email: element.email,
username: element.username,
fullName: element.fullName,
status: resultT
};
//console.log(resultT);
foundUsers.push(oneUser);
} else {
//Not found means not added or pending
var oneUser = {
email: element.email,
username: element.username,
fullName: element.fullName,
status: 0
};
foundUsers.push(oneUser);
//console.log(emailGiven + " " + element.email)
console.log(oneUser);
}
}
});
});
i have an object array for each elemant i would like to do mongoDB call for each element and depending on the results i wanna push the results in an array as im doing, the problem is that mongoDb is async so my main thread finished before i can push results to the array foundUsers, how may i fix this issue?

As you said, need to do handle an asynchronous operation into a synchronous loop. For doing this, you can use async library. It is so useful in such operatinos.
Just install async module in your project first
npm install --save async
Afterwards, you can do sth like this:
// for use with Node-style callbacks...
var async = require("async");
var obj = {dev: "/dev.json", test: "/test.json", prod: "/prod.json"};
var configs = {};
async.forEachOf(obj, (value, key, callback) => {
fs.readFile(__dirname + value, "utf8", (err, data) => {
if (err) return callback(err);
try {
configs[key] = JSON.parse(data);
} catch (e) {
return callback(e);
}
callback();
});
}, err => {
if (err) console.error(err.message);
// configs is now a map of JSON data
doSomethingWith(configs);
});
For working with this library, it uses async.forEachOf function instead of simple forEach loop. Three parameters is sent to this function.
The 1st parameter that is passed to async.forEachOf is an array to iterate over it (obj).
The 2nd parameter is a callback function that apply over each item in obj.
The 3rd or the last parameter that is passed to async.forEachOf function, is another callback function too. It is called when iteration process over every item in obj has finished.

Related

page renders before getting all the values sorted

I think the rendering takes place before the searching of the string on the files, i have tried different methods but don't seems to get this working. any help will be appreciated. im a noob on to the nodejs. im trying to get the id of the user and query and get all the data and there after see if he is in any of the lists given and finally render the page.
const j = [];
let name = '';
const filename = [];
var ext = '';
module.exports = function(app, express) {
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
app.post('/cusdetails', isLoggedIn, function (req, res) {
var cusid=req.body.cusid;
var insertQuerys = "SELECT * FROM customer WHERE cusid=? ORDER BY rowid DESC LIMIT 1";
connection.query(insertQuerys,[cusid],
function(err, rows){
rows.forEach( (row) => {
name=row.fncus;
});
fs.readdir('./views/iplist', function(err, files) {
if (err)
throw err;
for (var index in files) {
j.push(files[index])
}
j.forEach(function(value) {
var k = require('path').resolve(__dirname, '../views/iplist/',value);
fs.exists(k, function(fileok){
if(fileok) {
fs.readFile(k, function(err, content) {
if (err) throw err;
if (content.indexOf(name) > -1) {
ext = path.extname(k);
filename.push(path.basename(k, ext));
}
});
}
else {
console.log(" FileNotExist ");
}
});
});
});
console.log(filename);
res.render('cusdetails.ejs', {rows: rows, user:req.user , aml: filename });
});
})
You can create simple Promise wrapper and then use it inside async/await function to pause execution until resolved.
// use mysql2 package as it provides promise, less work to write promise wrappers
const mysql = require('mysql2/promise');
// create the connection to database
const connection = mysql.createConnection({
host: 'localhost',
user: 'root',
database: 'test'
});
// sample wrapper
function some(k) {
// more advisable to have local variables, why do you need this to be array?
var filename = [];
return new Promise((resolve, reject) => {
// doing this is also not recommended check nodejs documentation **fs.exists** for more info
fs.exists(k, function(fileok){
if(fileok) {
fs.readFile(k, function(err, content) {
if (err) reject(err);
if (content.indexOf(name) > -1) {
ext = path.extname(k);
filename.push(path.basename(k, ext));
resolve(filename)
}
});
}
else {
// reject(new Error("FileNotExist"))
console.log(" FileNotExist ");
}
});
})
}
// note the use of async
app.post('/cusdetails', isLoggedIn, async function (req, res) {
var cusid=req.body.cusid;
var insertQuerys = "SELECT * FROM customer WHERE cusid=? ORDER BY rowid DESC LIMIT 1";
// using await to pause excution, waits till query is finished
const [rows] = await connection.query(insertQuerys,[cusid])
rows.forEach( (row) => {
name=row.fncus;
});
// then you can
var result = await some(k)
...
Note however this way you loose the advantage of concurrent execution, as it's kindoff blocking. If the result of one call is not used in another, you can execute in parallel and await for result to achieve sequencing like
const [rows] = connection.query(insertQuerys,[cusid])
var result = some(k)
console.log(await rows) // do something
console.log(await result) // do something
JavaScript is asynchronous. This means that if you have a function with a callback (i.e. your query), the callback will be called asynchronously, at an unknown time, while the other code executes.
You need to look up some tutorials how to deal with callbacks, to get a proper understanding of it. Another method is using async/await and/or promises.
Basically, if you take the following code:
console.log("this will print first");
setTimeout(function () {
console.log("this will print last");
}, 1000);
console.log("this will print second");
If you run the code above, the top level is executed synchronously, so, it first calls console.log, then it executes setTimeout, which is synchronous. It sets a timeout, then says "I'm ready", and the code continues to the other console.log. After 1 second (1000 milliseconds), the callback in the setTimeout function is executed, and only then that console.log is called. You can not make the rest of the code wait this way, you need to restructure your code or read into promises.

async foreach inside async series nodejs

I'm working on node async library. I'm not able to execute in the sequence i want. i don't know where I'm going wrong
here is the code.. in comments i have defined order number..
currently its executing in 2,3,4,5,1 order i want in 1,2,3,4,5 order ....kindly help
function getAsExhibitors(req, res) {
//getting all exhibitors against an event
var exhibitors = [];
var eac_app_names = [];
async.series([function(callback){
models.EacExhibitorsExt.find({ deleted: false,userid: req.user._id}).sort({ modified: -1 }).exec(function(err, myExhibitors) {
exhibitors = myExhibitors;
callback();
});
},function(callback){
async.forEach(exhibitors,function(exhibitor,callback){
models.Eac.findById(exhibitor.eventid).exec(function(err,eac){
eac_app_names[exhibitors.indexOf(exhibitor)]=eac;
console.log("-----------------1--------------"+eac_app_names);
});
console.log("-----------------2--------------"+eac_app_names);
callback();
},function(err) {
console.log("-----------------3--------------"+eac_app_names);
callback();
});
}],function(err) { //This function gets called after the two tasks have called their "task callbacks"
if (err) return next(err);
//Here locals will be populated with 'exhibitors' and 'apps'
console.log("-------------------------4------"+eac_app_names);
console.log("-------------------------5------"+eac_app_names.name);
res.locals.exhibitors = exhibitors;
res.locals.eac_app_names = eac_app_names;
res.render('eac/eac_reg_as_exhibitor', { title: "My Event Exhibitors", asexhibitor: exhibitors,app_names:eac_app_names});
});
};
All mongoose methods work as asynchronous.In your scenario try this way:
function getAsExhibitors(req, res) {
//getting all exhibitors against an event
var exhibitors = [];
var eac_app_names = [];
async.series([function(callback){
models.EacExhibitorsExt.find({ deleted: false,userid: req.user._id}).sort({ modified: -1 }).exec(function(err, myExhibitors) {
exhibitors = myExhibitors;
callback();
});
},function(callback){
async.forEach(exhibitors,function(exhibitor,callback){
models.Eac.findById(exhibitor.eventid).exec(function(err,eac){
eac_app_names[exhibitors.indexOf(exhibitor)]=eac;
console.log("-----------------1--------------"+eac_app_names);
console.log("-----------------2--------------"+eac_app_names);
callback();
});
},function(err) {
console.log("-----------------3--------------"+eac_app_names);
callback();
});
}],function(err) { //This function gets called after the two tasks have called their "task callbacks"
if (err) return next(err);
//Here locals will be populated with 'exhibitors' and 'apps'
console.log("-------------------------4------"+eac_app_names);
console.log("-------------------------5------"+eac_app_names.name);
res.locals.exhibitors = exhibitors;
res.locals.eac_app_names = eac_app_names;
res.render('eac/eac_reg_as_exhibitor', { title: "My Event Exhibitors", asexhibitor: exhibitors,app_names:eac_app_names});
});
};
You are welcome to use es6 with generator.
Try out co-foreach-series to get each of the array element and execute async function one by one.
ForEach Series Example
foreach(yourArray, function(element, index) {
// Each of this function will be executed one after one
co(function*() {
// Do some async task, and wait until this task be finished
yield yourAsyncFunc();
yield doOtherAsyncTasks();
})
})

Make Node.js code synchronous in Mongoose while iterating

I am learning Node.js; due to asynchronous of Node.js I am facing an issue:
domain.User.find({userName: new RegExp(findtext, 'i')}).sort('-created').skip(skip).limit(limit)
.exec(function(err, result) {
for(var i=0;i<result.length;i++){
console.log("result is ",result[i].id);
var camera=null;
domain.Cameras.count({"userId": result[i].id}, function (err, cameraCount) {
if(result.length-1==i){
configurationHolder.ResponseUtil.responseHandler(res, result, "User List ", false, 200);
}
})
}
})
I want to use result in Cameras callback but it is empty array here, so is there anyway to get it?
And this code is asynchronous, is it possible if we make a complete function synchronous?
#jmingov is right. You should make use of the async module to execute parallel requests to get the counts for each user returned in the User.find query.
Here's a flow for demonstration:
var Async = require('async'); //At the top of your js file.
domain.User.find({userName: new RegExp(findtext, 'i')}).sort('-created').skip(skip).limit(limit)
.exec(function(err, result) {
var cameraCountFunctions = [];
result.forEach(function(user) {
if (user && user.id)
{
console.log("result is ", user.id);
var camera=null; //What is this for?
cameraCountFunctions.push( function(callback) {
domain.Cameras.count({"userId": user.id}, function (err, cameraCount) {
if (err) return callback(err);
callback(null, cameraCount);
});
});
}
})
Async.parallel(cameraCountFunctions, function (err, cameraCounts) {
console.log(err, cameraCounts);
//CameraCounts is an array with the counts for each user.
//Evaluate and return the results here.
});
});
Try to do async programing allways when doing node.js, this is a must. Or youll end with big performance problems.
Check this module: https://github.com/caolan/async it can help.
Here is the trouble in your code:
domain.Cameras.count({
"userId": result[i].id
}, function(err, cameraCount) {
// the fn() used in the callback has 'cameraCount' as argument so
// mongoose will store the results there.
if (cameraCount.length - 1 == i) { // here is the problem
// result isnt there it should be named 'cameraCount'
configurationHolder.ResponseUtil.responseHandler(res, cameraCount, "User List ", false, 200);
}
});

module.export function not returning results

I'm having some problems returning results from a module function.
Below are two files that I'm working with.
When I call the exported function it returns nothings.
Any suggestions/fixes as to why? Does it have to do with callbacks?
models/index.js
module.exports = exports = function(library) {
modCodes.findOne({name: library}, {modcode:1}, function(err, mc) {
if (err) throw new Error(err);
var db = mongoose.createConnection('mongodb://localhost:27017/' + mc.modcode + '?safe=true');
var models = {
Books: db.model('books', require('./schemas/books'))
}
return models;
});
};
books.js
var Models = require('../models');
console.log(Models("myLibrary")); //return nothing
The reason you're getting no results is that you're trying to return a function value synchronously from an asynchronous callback. Instead of providing a function value, the return statement will instead stop the function, as return; would normally do. This is why you must use a callback for asynchronous operations:
module.exports = exports = function(library, callback) {
modCodes.findOne({name: library}, {modcode: 1}, function (err, mc) {
if (err) throw new Error(err);
var db = mongoose.createConnection('mongodb://localhost:27017/' + mc.modcode + '?safe=true');
var models = {
Books: db.model('books', require('./schemas/books'))
}
callback(models);
});
};
And this is how you would be able to use it:
var Models = require('../models');
Models('myLibrary', function(models) {
console.log(models);
});
i solve similar problem in different way. I am not sure whether it is right way.
in main node js I am using a model named product. I am passing product and res to misc.js. Following is part of my server.js file
var misc = require('./misc');
app.get('/groupbyCategory', function(req,res,next)
{
var res2;
misc.addX(product,res);
})
IN misc.js doing group by function and will return that value to straight way to angular controller. it is not necessary to return the result to server.js and from server.js to return angular controller. So i feel waiting and other call back seems unnecessary.
Inside misc.js i keep following code.
exports.addX = function(product,res) {
product.aggregate([
{ $group: {
_id: {category: "$category"},
count: { $sum: 1 }
}}
], function (err, result) {
if (err) {
console.log(err);
return err;
}
else
{
//return result;
console.log(result);
res.send(result);
}
});
};

Iterating over a mongodb cursor serially (waiting for callbacks before moving to next document)

Using mongoskin, I can do a query like this, which will return a cursor:
myCollection.find({}, function(err, resultCursor) {
resultCursor.each(function(err, result) {
}
}
However, I'd like to call some async functions for each document, and only move on to the next item on the cursor after this has called back (similar to the eachSeries structure in the async.js module). E.g:
myCollection.find({}, function(err, resultCursor) {
resultCursor.each(function(err, result) {
externalAsyncFunction(result, function(err) {
//externalAsyncFunction completed - now want to move to next doc
});
}
}
How could I do this?
Thanks
UPDATE:
I don't wan't to use toArray() as this is a large batch operation, and the results might not fit in memory in one go.
A more modern approach that uses async/await:
const cursor = db.collection("foo").find({});
while(await cursor.hasNext()) {
const doc = await cursor.next();
// process doc here
}
Notes:
This may be even more simple to do when async iterators arrive.
You'll probably want to add try/catch for error checking.
The containing function should be async or the code should be wrapped in (async function() { ... })() since it uses await.
If you want, add await new Promise(resolve => setTimeout(resolve, 1000)); (pause for 1 second) at the end of the while loop to show that it does process docs one after the other.
If you don't want to load all of the results into memory using toArray, you can iterate using the cursor with something like the following.
myCollection.find({}, function(err, resultCursor) {
function processItem(err, item) {
if(item === null) {
return; // All done!
}
externalAsyncFunction(item, function(err) {
resultCursor.nextObject(processItem);
});
}
resultCursor.nextObject(processItem);
}
since node.js v10.3 you can use async iterator
const cursor = db.collection('foo').find({});
for await (const doc of cursor) {
// do your thing
// you can even use `await myAsyncOperation()` here
}
Jake Archibald wrote a great blog post about async iterators, that I came to know after reading #user993683's answer.
This works with large dataset by using setImmediate:
var cursor = collection.find({filter...}).cursor();
cursor.nextObject(function fn(err, item) {
if (err || !item) return;
setImmediate(fnAction, item, arg1, arg2, function() {
cursor.nextObject(fn);
});
});
function fnAction(item, arg1, arg2, callback) {
// Here you can do whatever you want to do with your item.
return callback();
}
If someone is looking for a Promise way of doing this (as opposed to using callbacks of nextObject), here it is. I am using Node v4.2.2 and mongo driver v2.1.7. This is kind of an asyncSeries version of Cursor.forEach():
function forEachSeries(cursor, iterator) {
return new Promise(function(resolve, reject) {
var count = 0;
function processDoc(doc) {
if (doc != null) {
count++;
return iterator(doc).then(function() {
return cursor.next().then(processDoc);
});
} else {
resolve(count);
}
}
cursor.next().then(processDoc);
});
}
To use this, pass the cursor and an iterator that operates on each document asynchronously (like you would for Cursor.forEach). The iterator needs to return a promise, like most mongodb native driver functions do.
Say, you want to update all documents in the collection test. This is how you would do it:
var theDb;
MongoClient.connect(dbUrl).then(function(db) {
theDb = db; // save it, we'll need to close the connection when done.
var cur = db.collection('test').find();
return forEachSeries(cur, function(doc) { // this is the iterator
return db.collection('test').updateOne(
{_id: doc._id},
{$set: {updated: true}} // or whatever else you need to change
);
// updateOne returns a promise, if not supplied a callback. Just return it.
});
})
.then(function(count) {
console.log("All Done. Processed", count, "records");
theDb.close();
})
You can do something like this using the async lib. The key point here is to check if the current doc is null. If it is, it means you are finished.
async.series([
function (cb) {
cursor.each(function (err, doc) {
if (err) {
cb(err);
} else if (doc === null) {
cb();
} else {
console.log(doc);
array.push(doc);
}
});
}
], function (err) {
callback(err, array);
});
You could use a Future:
myCollection.find({}, function(err, resultCursor) {
resultCursor.count(Meteor.bindEnvironment(function(err,count){
for(var i=0;i<count;i++)
{
var itemFuture=new Future();
resultCursor.nextObject(function(err,item)){
itemFuture.result(item);
}
var item=itemFuture.wait();
//do what you want with the item,
//and continue with the loop if so
}
}));
});
You can get the result in an Array and iterate using a recursive function, something like this.
myCollection.find({}).toArray(function (err, items) {
var count = items.length;
var fn = function () {
externalAsyncFuntion(items[count], function () {
count -= 1;
if (count) fn();
})
}
fn();
});
Edit:
This is only applicable for small datasets, for larger one's you should use cursors as mentioned in other answers.
A more modern approach that uses for await:
const cursor = db.collection("foo").find({});
for await(const doc of cursor) {
// process doc here with await
await processDoc(doc);
}
You could use simple setTimeOut's. This is an example in typescript running on nodejs (I am using promises via the 'when' module but it can be done without them as well):
import mongodb = require("mongodb");
var dbServer = new mongodb.Server('localhost', 27017, {auto_reconnect: true}, {});
var db = new mongodb.Db('myDb', dbServer);
var util = require('util');
var when = require('when'); //npm install when
var dbDefer = when.defer();
db.open(function() {
console.log('db opened...');
dbDefer.resolve(db);
});
dbDefer.promise.then(function(db : mongodb.Db){
db.collection('myCollection', function (error, dataCol){
if(error) {
console.error(error); return;
}
var doneReading = when.defer();
var processOneRecordAsync = function(record) : When.Promise{
var result = when.defer();
setTimeout (function() {
//simulate a variable-length operation
console.log(util.inspect(record));
result.resolve('record processed');
}, Math.random()*5);
return result.promise;
}
var runCursor = function (cursor : MongoCursor){
cursor.next(function(error : any, record : any){
if (error){
console.log('an error occurred: ' + error);
return;
}
if (record){
processOneRecordAsync(record).then(function(r){
setTimeout(function() {runCursor(cursor)}, 1);
});
}
else{
//cursor up
doneReading.resolve('done reading data.');
}
});
}
dataCol.find({}, function(error, cursor : MongoCursor){
if (!error)
{
setTimeout(function() {runCursor(cursor)}, 1);
}
});
doneReading.promise.then(function(message : string){
//message='done reading data'
console.log(message);
});
});
});

Resources