module.export function not returning results - node.js

I'm having some problems returning results from a module function.
Below are two files that I'm working with.
When I call the exported function it returns nothings.
Any suggestions/fixes as to why? Does it have to do with callbacks?
models/index.js
module.exports = exports = function(library) {
modCodes.findOne({name: library}, {modcode:1}, function(err, mc) {
if (err) throw new Error(err);
var db = mongoose.createConnection('mongodb://localhost:27017/' + mc.modcode + '?safe=true');
var models = {
Books: db.model('books', require('./schemas/books'))
}
return models;
});
};
books.js
var Models = require('../models');
console.log(Models("myLibrary")); //return nothing

The reason you're getting no results is that you're trying to return a function value synchronously from an asynchronous callback. Instead of providing a function value, the return statement will instead stop the function, as return; would normally do. This is why you must use a callback for asynchronous operations:
module.exports = exports = function(library, callback) {
modCodes.findOne({name: library}, {modcode: 1}, function (err, mc) {
if (err) throw new Error(err);
var db = mongoose.createConnection('mongodb://localhost:27017/' + mc.modcode + '?safe=true');
var models = {
Books: db.model('books', require('./schemas/books'))
}
callback(models);
});
};
And this is how you would be able to use it:
var Models = require('../models');
Models('myLibrary', function(models) {
console.log(models);
});

i solve similar problem in different way. I am not sure whether it is right way.
in main node js I am using a model named product. I am passing product and res to misc.js. Following is part of my server.js file
var misc = require('./misc');
app.get('/groupbyCategory', function(req,res,next)
{
var res2;
misc.addX(product,res);
})
IN misc.js doing group by function and will return that value to straight way to angular controller. it is not necessary to return the result to server.js and from server.js to return angular controller. So i feel waiting and other call back seems unnecessary.
Inside misc.js i keep following code.
exports.addX = function(product,res) {
product.aggregate([
{ $group: {
_id: {category: "$category"},
count: { $sum: 1 }
}}
], function (err, result) {
if (err) {
console.log(err);
return err;
}
else
{
//return result;
console.log(result);
res.send(result);
}
});
};

Related

Mongoose Promise not populating in script

I have an api server and some script jobs. They are using the same function to pull a roster using mongoose and populate the players in the roster.
On the api server, this function is called normally. Using the script, it doesn't.
API example
function getRoster(id) {
var deferred = Q.defer();
Roster.find({_id:id}, 'playerRoster userId tournamentId').populate('playerRoster').exec(
function(err, roster) {
if (err) {
deferred.resolve(err);
}
deferred.resolve(roster[0]);
});
return deferred.promise;
}
api.post('/get_roster', function(req, res) {
// get tournament
var id = req.body._id;
var playerId = req.body.playerId;
getRoster(id).then(function(data) {
var roster=data;
res.json(roster);
});
});
Script
module.exports = function(config) {
this.getRoster=function(id) {
//return Roster.find({_id:id}, 'playerRoster userId tournamentId').exec( - THIS RETURNS
return Roster.find({_id:id}, 'playerRoster userId tournamentId').populate('playerRoster').exec(
function(err, roster) {
if (err) {
return err;
}
console.log('roster[0]',roster);
return roster[0];
});
}
this.tallyPoints = function(tournamentPlayer,sportsPlayers) {
var deferred = Q.defer();
var totalPoints =0;
console.log("tallyPoints 0 ",tournamentPlayer);
var rosterId = tournamentPlayer.player.roster[0];
console.log("tallyPoints 1 ",rosterId);
this.getRoster(rosterId).then(function(roster2){
console.log("tallyPoints 2 ",roster2);
...
deferred.resolve(totalPoints);
});
return deferred.promise;
}
return this;
};
In the script, neither logging for the roster[0] or tallyPoints 2 lines print, but there is no error either.
Why doesn't Roster.find return when I add populate? The only thing I can imagine is because playerRoster collection has 2000 records searching for ~10 and it hits some timeout that isn't being caught.
Any suggestion to clean it up is also appreciated.
Thanks
Moongoose supports promises for a long time. It's unsuitable to use callback-based Mongoose API where promises are desirable and the use of Q.defer with existing promises is known as Deferred antipattern (similarly, new Promise results in promise construction antipattern).
In its current state getRoster doesn't return a promise and doesn't handle errors correctly.
function getRoster(id) {
return Roster.find({_id:id}, 'playerRoster userId tournamentId').populate('playerRoster').exec()
.then(roster => roster[0]);
}
api.post('/get_roster', function(req, res) {
// get tournament
var id = req.body._id;
var playerId = req.body.playerId;
getRoster(id)
.then(function(data) {
var roster=data;
res.json(roster);
})
.catch(err => {
// handle error
});
});
Considering that only roster[0] is used, it likely should be changed to Roster.findOne.
It doesn't matter whether getRoster is used in Express route or elsewhere, it should work. It's unknown how module.exports = function(config) {...} module is used, but this may refer to wrong context if it isn't used as class. If getRoster and tallyPoints don't use config, they shouldn't reside inside this function.

Foreach loop with mongoDB call / node.js

result.forEach(element => {
//Get each element
console.log("LOOP");
dbo.collection("users").findOne({email: emailGiven, "friends.email": element.email},function(errT, resultT) {
if (errT){
console.log("Query Error Inside!");
res.status(errT.status); // or use err.statusCode instead
console.log(errT);
//db.close();
//return res.send(errT.message);
}
else {
if (resultT) {
var oneUser = {
email: element.email,
username: element.username,
fullName: element.fullName,
status: resultT
};
//console.log(resultT);
foundUsers.push(oneUser);
} else {
//Not found means not added or pending
var oneUser = {
email: element.email,
username: element.username,
fullName: element.fullName,
status: 0
};
foundUsers.push(oneUser);
//console.log(emailGiven + " " + element.email)
console.log(oneUser);
}
}
});
});
i have an object array for each elemant i would like to do mongoDB call for each element and depending on the results i wanna push the results in an array as im doing, the problem is that mongoDb is async so my main thread finished before i can push results to the array foundUsers, how may i fix this issue?
As you said, need to do handle an asynchronous operation into a synchronous loop. For doing this, you can use async library. It is so useful in such operatinos.
Just install async module in your project first
npm install --save async
Afterwards, you can do sth like this:
// for use with Node-style callbacks...
var async = require("async");
var obj = {dev: "/dev.json", test: "/test.json", prod: "/prod.json"};
var configs = {};
async.forEachOf(obj, (value, key, callback) => {
fs.readFile(__dirname + value, "utf8", (err, data) => {
if (err) return callback(err);
try {
configs[key] = JSON.parse(data);
} catch (e) {
return callback(e);
}
callback();
});
}, err => {
if (err) console.error(err.message);
// configs is now a map of JSON data
doSomethingWith(configs);
});
For working with this library, it uses async.forEachOf function instead of simple forEach loop. Three parameters is sent to this function.
The 1st parameter that is passed to async.forEachOf is an array to iterate over it (obj).
The 2nd parameter is a callback function that apply over each item in obj.
The 3rd or the last parameter that is passed to async.forEachOf function, is another callback function too. It is called when iteration process over every item in obj has finished.

Node.js module to fetch data from MongoDB database

I want to use an module to get and process data from my MongoDB database. (It should generate an object that represents my Express.js site's navbar)
I thought of doing something like this:
var nav = { Home: "/" };
module.exports = function() {
MongoClient.connect(process.env.MONGO_URL, function(err, db) {
assert.equal(err, null);
fetchData(db, function(articles, categories) {
combine(articles, categories, function(sitemap) {
// I got the data. What now?
console.log("NAV: ", nav);
})
});
});
};
var fetchData = function(db, callback) {
db.collection('articles').find({}).toArray(function(err, result) {
assert.equal(err);
articles = result;
db.collection('categories').find({}).toArray(function(err, result) {
assert.equal(err);
categories = result;
db.close();
callback(articles, categories);
});
});
};
var combine = function(articles, categories, callback) {
categories.forEach(function(category) {
nav[category.title] = {};
articles.forEach(function(article) {
if(article.category == category.name) {
nav[category.title][article.title] = "link";
}
})
});
callback(nav);
};
As of line 6, I do have all data I need.
(An object, currenty like { Home: '/', Uncategorized: { 'Hello world!': 'link' } })
But since I'm in an anonymous function, I don't know how to return that value. I mean, return would just return it the function that called it... And in the end, MongoClient.connect would receive my data.
If I set a variable instead, it would be set as module.exports returned before Node can even query the data from the database, right?
What can I do in order to make this work?
It should result in some kind of function, like
var nav = require('nav');
console.log(nav());
Thanks in advance!
Add another callback:
var nav = { Home: "/" };
module.exports = function(cb) {
MongoClient.connect(process.env.MONGO_URL, function(err, db) {
assert.equal(err, null);
fetchData(db, function(articles, categories) {
combine(articles, categories, function(sitemap) {
cb(sitemap);
})
});
})
});
And then use this way:
var nav = require('nav');
nav(function(sitemap){ console.log(sitemap); });
You can use mongoose module or monk module. These modules have been tested properly .
Just use
npm install mongoose or monk
The suggestion about mongoose is great and you can look into it, however I think you've already done the job with the fetching of the data from the db. You just need to access it in your main node flow.
You can try this:
module.exports.generateNav = function() {
MongoClient.connect(process.env.MONGO_URL, function(err, db) {
assert.equal(err, null);
var output = fetchData(db, function(articles, categories) {
combine(articles, categories, function(sitemap) {
})
});
return (output);
});
};
And then in your main application you can call it in the following way:
var nav = require('nav');
navigation = nav.generateNav();
console.log(navigation);

Wait for validation (serverside) to complete befor insert into database

I am pretty new to Node.js or Javascript in general when it comes to serverside stuff. Currently I am tring to validate some of the user input and set default values if something is wrong. Now if I run my validation the json object appears in the database befor my validation is completed.
The way I am doing the validation isnt maybe the best right now but if someone can explain me the behavior, I am pretty sure i can understand Javascript alot better in the future.
Is there a better way of doing validation (without mongoose or other ODM modules) with callbacks, middleware or should I use some async module?
Here is my code:
module.exports = function(app, express, todoDB, listDB, statusDB) {
var moment = require('moment');
var todoRouter = express.Router();
todoRouter.post('/', function(req, res, next) {
console.log('1');
if (!(moment(req.body.createDate).isValid())) {
req.body.createDate = moment().format("DD-MM-YYYY HH:mm:ss");
}
else {
req.body.createDate = moment(req.body.createDate).format("DD-MM-YYYY HH:mm:ss");
}
console.log('2');
if (req.body.list_id == '') {
listDB.findOne({list: 'Neu'}, function(reqdb, docs) {
if (docs == null) {
listDB.insert({list: 'Neu', index: 1});
listDB.findOne({list: 'Neu'}, function(reqdb, docs) {
console.log('AnlageListID');
console.log(docs._id);
req.body.list_id = docs._id;
});
}
else {
console.log('BestehendeListID');
console.log(docs._id);
req.body.list_id = docs._id;
}
});
}
console.log('3');
if (req.body.status_id == '') {
statusDB.findOne({status: 'offen'}, function(reqdb, docs) {
if (docs == null) {
statusDB.insert({status: 'offen', index: 1});
statusDB.findOne({status: 'offen'}, function(reqdb, docs) {
console.log('AnlageStatusID');
console.log(docs._id);
req.body.status_id = docs._id;
});
}
else {
console.log('BestehendeStatusID');
console.log(docs._id)
req.body.status_id = docs._id;
}
});
}
console.log('4');
console.log('StatusID');
console.log(req.body.status_id);
console.log('ListID');
console.log(req.body.list_id);
todoDB.insert({
todo: req.body.todo,
createDate: req.body.createDate,
endDate: req.body.endDate,
discription: req.body.discription,
comment: req.body.comment,
list_id: req.body.list_id,
priority_id: req.body.priority_id,
section_id: req.body.section_id,
user_id: req.body.user_id,
status_id: req.body.status_id,
company_id: req.body.company_id
});
res.json({message: 'TODO erfolgreich hinzugefügt!'});
});
return todoRouter;
};
... and this is the ouput:
1
2
3
4
StatusID
ListID
POST /api/todos 200 76.136 ms - 44
BestehendeListID
M3Xh46VjVjaTFoCM
BestehendeStatusID
48v80B4fbO87c8um
PS: Its a small "project" just for me learing the MEAN Stack so I am using neDB.
If I understand correctly you try to sequentially execute a number of asynchronous calls and introduce checks in the code to validate if previous asynchronous calls have completed. This is not going to work in a general case because your checks may be processed before the asynchronous call goes through. It might work now and then just by chance, but I would not expect even that.
There are standard mechanisms for that. One of them is using promises, another one using async and yet another one if stacking up all callbacks one into another. Below I will demonstrate how to address the problem using async, but the same general idea applies to using promises. Check the async project on Github then the following part-solution will become clear:
var async = require("async")
async.waterfall([
function(next) {
listDB.findOne({list: 'Neu'}, next); // quits on error
},
function(doc, next) {
if (doc) {
return next(null, doc._id);
}
statusDB.insert({status: 'offen', index: 1}, function(err) {
if (err) return next(err); // quit on error
statusDB.findOne({status: 'offen'}, function(err, doc) {
next(err, doc._id); // quits on error
});
});
},
function(id, next) {
// do next step and so on
next();
}
],
// this is the exit function: it will get called whenever an error
// is passed to any of the `next` callbacks or when the last
// function in the waterfall series calls its `next` callback (with
// or without an error)
function(err) {
console.error("Error processing:", err)
});

Iterating over a mongodb cursor serially (waiting for callbacks before moving to next document)

Using mongoskin, I can do a query like this, which will return a cursor:
myCollection.find({}, function(err, resultCursor) {
resultCursor.each(function(err, result) {
}
}
However, I'd like to call some async functions for each document, and only move on to the next item on the cursor after this has called back (similar to the eachSeries structure in the async.js module). E.g:
myCollection.find({}, function(err, resultCursor) {
resultCursor.each(function(err, result) {
externalAsyncFunction(result, function(err) {
//externalAsyncFunction completed - now want to move to next doc
});
}
}
How could I do this?
Thanks
UPDATE:
I don't wan't to use toArray() as this is a large batch operation, and the results might not fit in memory in one go.
A more modern approach that uses async/await:
const cursor = db.collection("foo").find({});
while(await cursor.hasNext()) {
const doc = await cursor.next();
// process doc here
}
Notes:
This may be even more simple to do when async iterators arrive.
You'll probably want to add try/catch for error checking.
The containing function should be async or the code should be wrapped in (async function() { ... })() since it uses await.
If you want, add await new Promise(resolve => setTimeout(resolve, 1000)); (pause for 1 second) at the end of the while loop to show that it does process docs one after the other.
If you don't want to load all of the results into memory using toArray, you can iterate using the cursor with something like the following.
myCollection.find({}, function(err, resultCursor) {
function processItem(err, item) {
if(item === null) {
return; // All done!
}
externalAsyncFunction(item, function(err) {
resultCursor.nextObject(processItem);
});
}
resultCursor.nextObject(processItem);
}
since node.js v10.3 you can use async iterator
const cursor = db.collection('foo').find({});
for await (const doc of cursor) {
// do your thing
// you can even use `await myAsyncOperation()` here
}
Jake Archibald wrote a great blog post about async iterators, that I came to know after reading #user993683's answer.
This works with large dataset by using setImmediate:
var cursor = collection.find({filter...}).cursor();
cursor.nextObject(function fn(err, item) {
if (err || !item) return;
setImmediate(fnAction, item, arg1, arg2, function() {
cursor.nextObject(fn);
});
});
function fnAction(item, arg1, arg2, callback) {
// Here you can do whatever you want to do with your item.
return callback();
}
If someone is looking for a Promise way of doing this (as opposed to using callbacks of nextObject), here it is. I am using Node v4.2.2 and mongo driver v2.1.7. This is kind of an asyncSeries version of Cursor.forEach():
function forEachSeries(cursor, iterator) {
return new Promise(function(resolve, reject) {
var count = 0;
function processDoc(doc) {
if (doc != null) {
count++;
return iterator(doc).then(function() {
return cursor.next().then(processDoc);
});
} else {
resolve(count);
}
}
cursor.next().then(processDoc);
});
}
To use this, pass the cursor and an iterator that operates on each document asynchronously (like you would for Cursor.forEach). The iterator needs to return a promise, like most mongodb native driver functions do.
Say, you want to update all documents in the collection test. This is how you would do it:
var theDb;
MongoClient.connect(dbUrl).then(function(db) {
theDb = db; // save it, we'll need to close the connection when done.
var cur = db.collection('test').find();
return forEachSeries(cur, function(doc) { // this is the iterator
return db.collection('test').updateOne(
{_id: doc._id},
{$set: {updated: true}} // or whatever else you need to change
);
// updateOne returns a promise, if not supplied a callback. Just return it.
});
})
.then(function(count) {
console.log("All Done. Processed", count, "records");
theDb.close();
})
You can do something like this using the async lib. The key point here is to check if the current doc is null. If it is, it means you are finished.
async.series([
function (cb) {
cursor.each(function (err, doc) {
if (err) {
cb(err);
} else if (doc === null) {
cb();
} else {
console.log(doc);
array.push(doc);
}
});
}
], function (err) {
callback(err, array);
});
You could use a Future:
myCollection.find({}, function(err, resultCursor) {
resultCursor.count(Meteor.bindEnvironment(function(err,count){
for(var i=0;i<count;i++)
{
var itemFuture=new Future();
resultCursor.nextObject(function(err,item)){
itemFuture.result(item);
}
var item=itemFuture.wait();
//do what you want with the item,
//and continue with the loop if so
}
}));
});
You can get the result in an Array and iterate using a recursive function, something like this.
myCollection.find({}).toArray(function (err, items) {
var count = items.length;
var fn = function () {
externalAsyncFuntion(items[count], function () {
count -= 1;
if (count) fn();
})
}
fn();
});
Edit:
This is only applicable for small datasets, for larger one's you should use cursors as mentioned in other answers.
A more modern approach that uses for await:
const cursor = db.collection("foo").find({});
for await(const doc of cursor) {
// process doc here with await
await processDoc(doc);
}
You could use simple setTimeOut's. This is an example in typescript running on nodejs (I am using promises via the 'when' module but it can be done without them as well):
import mongodb = require("mongodb");
var dbServer = new mongodb.Server('localhost', 27017, {auto_reconnect: true}, {});
var db = new mongodb.Db('myDb', dbServer);
var util = require('util');
var when = require('when'); //npm install when
var dbDefer = when.defer();
db.open(function() {
console.log('db opened...');
dbDefer.resolve(db);
});
dbDefer.promise.then(function(db : mongodb.Db){
db.collection('myCollection', function (error, dataCol){
if(error) {
console.error(error); return;
}
var doneReading = when.defer();
var processOneRecordAsync = function(record) : When.Promise{
var result = when.defer();
setTimeout (function() {
//simulate a variable-length operation
console.log(util.inspect(record));
result.resolve('record processed');
}, Math.random()*5);
return result.promise;
}
var runCursor = function (cursor : MongoCursor){
cursor.next(function(error : any, record : any){
if (error){
console.log('an error occurred: ' + error);
return;
}
if (record){
processOneRecordAsync(record).then(function(r){
setTimeout(function() {runCursor(cursor)}, 1);
});
}
else{
//cursor up
doneReading.resolve('done reading data.');
}
});
}
dataCol.find({}, function(error, cursor : MongoCursor){
if (!error)
{
setTimeout(function() {runCursor(cursor)}, 1);
}
});
doneReading.promise.then(function(message : string){
//message='done reading data'
console.log(message);
});
});
});

Resources