It should be very awesome to use non-blocking code but I'm running out of ideas how to acomplish this task. I have to validate a value by making few db queries like so:
validate = function() {
var valid = true;
db.collection('posts').findOne({date: ....}, function(err, post){
if (..) valid = false
}
db.collection('posts').findOne({author: .....}, function(err, post){
if (..) valid = false
}
return valid;
}
It is very good that validations can run in concurent manner, but the problem is how to return the final state. Obviously my example will not work. The function will return before db queries execution.
Welcome to the async world.
You should use something like async or fnqueue for your control flow,
then you can setup a chain of validations.
function isValid (mainCallback) {
new FnQueue({
date: function (callback) {
if (...) {
callback();
} else {
callback(new Error('what\'s happened here');
}
},
author: function (callback) {
db.collection('posts').findOne({ author: ..... }, callback);
}
},
function (err, data) {
mainCallback(Boolean(err)); //you should more than this :)
},
1 // concurrency level for serial execution
);
If you are using mongoose, then you can use the validations that are supported in the models. Take a look the validation docs for details and examples.
If you are not using mongoose, then you will need to pass a callback to your validate function, and the callback will receive the boolean. Also, you will need to handle the flow of your function so that they are run in series or parallel, depending on your needs. So if it is in in series, the following would work:
validate = function(callback) {
var valid = true;
db.collection('posts').findOne({date: ....}, function(err, post){
if (..) {
return callback(true);
}
db.collection('posts').findOne({author: .....}, function(err, post){
if (..) callback(false);
});
});
}
Related
I have this file containing the following code which is the database layer of my api. It is externally dependent on SQL Server to fetch the data.
var sql = require('mssql');
var async = require('async');
module.exports = {
getDetails: function(number, callback) {
async.parallel({
general: function(callback) {
getGeneral(number, callback);
},
preferences: function(callback) {
getPref(number, callback);
}
},
function(err, results) {
if (err) {
logger.error(err);
throw err;
}
callback(results);
});
}
};
function getGeneral(number, callback) {
var mainconnection = new sql.Connection(dbCredentials[1].generalDBConfig, function(err) {
var request = new sql.Request(mainconnection);
request.input('number', sql.BigInt, number);
request.execute('[number].[genral_get]', function(err, recordsets) {
if (err) {
logger.error(err);
}
var gen = {};
var spResult = recordsets[0];
if (spResult[0] != null) {
spResult.forEach(function(record) {
var test = {};
gen[record.genCode] = record.genValue;
});
callback(null, gen);
} else {
callback(null, null);
}
});
});
}
function getPref(number, callback) {
var mainconnection = new sql.Connection(dbCredentials[0].prefDBConfig, function(err) {
var request = new sql.Request(mainconnection);
request.input('number', sql.BigInt, number);
request.execute('[number].[pref_get]', function(err, recordsets) {
if (err) {
logger.error(err);
}
var spResult = recordsets[0];
if (spResult[0] != null) {
callback(null, spResult[0]);
} else {
callback(null, null);
}
});
});
}
The database layer return this JSON format data:
{
"general": {
"number": "72720604"
"title": "Mr ",
"buildingNameNumber": null,
"subBuildingName": null,
"streetName": null,
"postalTown": null,
"county": null
},
"pref": {
"UseAccessibilitySite": "00000",
"IntroductorySource": "75"
}
};
As I am new to unit testing, I don't know how to start about writing unit tests for this module even though choosing mocha with chai as my unit testing framework. Any kind of suggestions or help is appreciated...
The question is: what exactly do you want to test in this code?
I'm assuming that the answer roughly is "I want to call getDetails() in my test suite and verify that it behaves correctly".
Of course, you don't want to create a whole MSSQL server in your test suite. Instead, it's much easier to stub the database.
A module for mocking, stubbing and spying that I've had great success with is sinon.js. For your code, you'll need to stub a few things:
Stub sql.Connection() to return a stubbed connection object.
Stub sql.Request() to return a stub object that has methods input and execute.
Stub sql.BigInt() to verify that it was called correctly.
Then, for the stubbed request object, you need to:
Verify that .input() was called correctly.
Make sure that execute returns the record(s) of your choosing.
It will be a lot of setup work to test the getDetails() function entirely. You might also be interested in the rewire module, which allows you to test getGeneral() and getPref() directly without having to add them to module.exports.
Finally, refactoring this code into smaller pieces will also help a lot. For example, if you could do something like this:
// This is your own db module, which takes care of connecting
// to the database for you.
var db = require('./db');
function getGeneral(number, callback) {
var request = db.createRequest();
// The rest of the function
}
Then it would be much easier to use sinon to create a fake request object that does exactly what you want it to do.
I am pretty new to Node.js or Javascript in general when it comes to serverside stuff. Currently I am tring to validate some of the user input and set default values if something is wrong. Now if I run my validation the json object appears in the database befor my validation is completed.
The way I am doing the validation isnt maybe the best right now but if someone can explain me the behavior, I am pretty sure i can understand Javascript alot better in the future.
Is there a better way of doing validation (without mongoose or other ODM modules) with callbacks, middleware or should I use some async module?
Here is my code:
module.exports = function(app, express, todoDB, listDB, statusDB) {
var moment = require('moment');
var todoRouter = express.Router();
todoRouter.post('/', function(req, res, next) {
console.log('1');
if (!(moment(req.body.createDate).isValid())) {
req.body.createDate = moment().format("DD-MM-YYYY HH:mm:ss");
}
else {
req.body.createDate = moment(req.body.createDate).format("DD-MM-YYYY HH:mm:ss");
}
console.log('2');
if (req.body.list_id == '') {
listDB.findOne({list: 'Neu'}, function(reqdb, docs) {
if (docs == null) {
listDB.insert({list: 'Neu', index: 1});
listDB.findOne({list: 'Neu'}, function(reqdb, docs) {
console.log('AnlageListID');
console.log(docs._id);
req.body.list_id = docs._id;
});
}
else {
console.log('BestehendeListID');
console.log(docs._id);
req.body.list_id = docs._id;
}
});
}
console.log('3');
if (req.body.status_id == '') {
statusDB.findOne({status: 'offen'}, function(reqdb, docs) {
if (docs == null) {
statusDB.insert({status: 'offen', index: 1});
statusDB.findOne({status: 'offen'}, function(reqdb, docs) {
console.log('AnlageStatusID');
console.log(docs._id);
req.body.status_id = docs._id;
});
}
else {
console.log('BestehendeStatusID');
console.log(docs._id)
req.body.status_id = docs._id;
}
});
}
console.log('4');
console.log('StatusID');
console.log(req.body.status_id);
console.log('ListID');
console.log(req.body.list_id);
todoDB.insert({
todo: req.body.todo,
createDate: req.body.createDate,
endDate: req.body.endDate,
discription: req.body.discription,
comment: req.body.comment,
list_id: req.body.list_id,
priority_id: req.body.priority_id,
section_id: req.body.section_id,
user_id: req.body.user_id,
status_id: req.body.status_id,
company_id: req.body.company_id
});
res.json({message: 'TODO erfolgreich hinzugefügt!'});
});
return todoRouter;
};
... and this is the ouput:
1
2
3
4
StatusID
ListID
POST /api/todos 200 76.136 ms - 44
BestehendeListID
M3Xh46VjVjaTFoCM
BestehendeStatusID
48v80B4fbO87c8um
PS: Its a small "project" just for me learing the MEAN Stack so I am using neDB.
If I understand correctly you try to sequentially execute a number of asynchronous calls and introduce checks in the code to validate if previous asynchronous calls have completed. This is not going to work in a general case because your checks may be processed before the asynchronous call goes through. It might work now and then just by chance, but I would not expect even that.
There are standard mechanisms for that. One of them is using promises, another one using async and yet another one if stacking up all callbacks one into another. Below I will demonstrate how to address the problem using async, but the same general idea applies to using promises. Check the async project on Github then the following part-solution will become clear:
var async = require("async")
async.waterfall([
function(next) {
listDB.findOne({list: 'Neu'}, next); // quits on error
},
function(doc, next) {
if (doc) {
return next(null, doc._id);
}
statusDB.insert({status: 'offen', index: 1}, function(err) {
if (err) return next(err); // quit on error
statusDB.findOne({status: 'offen'}, function(err, doc) {
next(err, doc._id); // quits on error
});
});
},
function(id, next) {
// do next step and so on
next();
}
],
// this is the exit function: it will get called whenever an error
// is passed to any of the `next` callbacks or when the last
// function in the waterfall series calls its `next` callback (with
// or without an error)
function(err) {
console.error("Error processing:", err)
});
I am learning Node.js; due to asynchronous of Node.js I am facing an issue:
domain.User.find({userName: new RegExp(findtext, 'i')}).sort('-created').skip(skip).limit(limit)
.exec(function(err, result) {
for(var i=0;i<result.length;i++){
console.log("result is ",result[i].id);
var camera=null;
domain.Cameras.count({"userId": result[i].id}, function (err, cameraCount) {
if(result.length-1==i){
configurationHolder.ResponseUtil.responseHandler(res, result, "User List ", false, 200);
}
})
}
})
I want to use result in Cameras callback but it is empty array here, so is there anyway to get it?
And this code is asynchronous, is it possible if we make a complete function synchronous?
#jmingov is right. You should make use of the async module to execute parallel requests to get the counts for each user returned in the User.find query.
Here's a flow for demonstration:
var Async = require('async'); //At the top of your js file.
domain.User.find({userName: new RegExp(findtext, 'i')}).sort('-created').skip(skip).limit(limit)
.exec(function(err, result) {
var cameraCountFunctions = [];
result.forEach(function(user) {
if (user && user.id)
{
console.log("result is ", user.id);
var camera=null; //What is this for?
cameraCountFunctions.push( function(callback) {
domain.Cameras.count({"userId": user.id}, function (err, cameraCount) {
if (err) return callback(err);
callback(null, cameraCount);
});
});
}
})
Async.parallel(cameraCountFunctions, function (err, cameraCounts) {
console.log(err, cameraCounts);
//CameraCounts is an array with the counts for each user.
//Evaluate and return the results here.
});
});
Try to do async programing allways when doing node.js, this is a must. Or youll end with big performance problems.
Check this module: https://github.com/caolan/async it can help.
Here is the trouble in your code:
domain.Cameras.count({
"userId": result[i].id
}, function(err, cameraCount) {
// the fn() used in the callback has 'cameraCount' as argument so
// mongoose will store the results there.
if (cameraCount.length - 1 == i) { // here is the problem
// result isnt there it should be named 'cameraCount'
configurationHolder.ResponseUtil.responseHandler(res, cameraCount, "User List ", false, 200);
}
});
This is my code:
var thisValue = new models.Value({
id:id,
title:title //this is a unique value
});
console.log(thisValue);
thisValue.save(function(err, product, numberAffected) {
if (err) {
if (err.code === 11000) { //error for dupes
console.error('Duplicate blocked!');
models.Value.find({title:title}, function(err, docs)
{
callback(docs) //this is ugly
});
}
return;
}
console.log('Value saved:', product);
if (callback) {
callback(product);
}
});
If I detect that a duplicate is trying to be inserted, i block it. However, when that happens, i want to return the existing document. As you can see I have implemented a string of callbacks, but this is ugly and its unpredictable (ie. how do i know which callback will be called? How do i pass in the right one?). Does anyone know how to solve this problem? Any help appreciated.
While your code doesn't handle a few error cases, and uses the wrong find function, the general flow is typical giving the work you want to do.
If there are errors other than the duplicate, the callback isn't called, which likely will cause downstream issues in your NodeJs application
use findOne rather than find as there will be only one result given the key is unique. Otherwise, it will return an array.
If your callback expected the traditional error as the first argument, you could directly pass the callback to the findOne function rather than introducing an anonymous function.
You also might want to look at findOneAndUpdate eventually, depending on what your final schema and logic will be.
As mentioned, you might be able to use findOneAndUpdate, but with additional cost.
function save(id, title, callback) {
Value.findOneAndUpdate(
{id: id, title: title}, /* query */
{id: id, title: title}, /* update */
{ upsert: true}, /* create if it doesn't exist */
callback);
}
There's still a callback of course, but it will write the data again if the duplicate is found. Whether that's an issue is really dependent on use cases.
I've done a little clean-up of your code... but it's really quite simple and the callback should be clear. The callback to the function always receives either the newly saved document or the one that was matched as a duplicate. It's the responsibility of the function calling saveNewValue to check for an error and properly handle it. You'll see how I've also made certain that the callback is called regardless of type of error and is always called with the result in a consistent way.
function saveNewValue(id, title, callback) {
if (!callback) { throw new Error("callback required"); }
var thisValue = new models.Value({
id:id,
title:title //this is a unique value
});
thisValue.save(function(err, product) {
if (err) {
if (err.code === 11000) { //error for dupes
return models.Value.findOne({title:title}, callback);
}
}
callback(err, product);
});
}
Alternatively, you could use the promise pattern. This example is using when.js.
var when = require('when');
function saveNewValue(id, title) {
var deferred = when.defer();
var thisValue = new models.Value({
id:id,
title:title //this is a unique value
});
thisValue.save(function(err, product) {
if (err) {
if (err.code === 11000) { //error for dupes
return models.Value.findOne({title:title}, function(err, val) {
if (err) {
return deferred.reject(err);
}
return deferred.resolve(val);
});
}
return deferred.reject(err);
}
return deferred.resolve(product);
});
return deferred.promise;
}
saveNewValue('123', 'my title').then(function(doc) {
// success
}, function(err) {
// failure
});
I really like WiredPrairie's answer, but his promise implementation is way too complicated.
So, I decided to add my own promise implementation.
Mongoose 3.8.x
If you're using latest Mongoose 3.8.x then there is no need to use any other promise module, because since 3.8.0 model .create() method returns a promise:
function saveNewValue(id, title) {
return models.Value.create({
id:id,
title:title //this is a unique value
}).then(null, function(err) {
if (err.code === 11000) {
return models.Value.findOne({title:title}).exec()
} else {
throw err;
}
});
}
saveNewValue('123', 'my title').then(function(doc) {
// success
console.log('success', doc);
}, function(err) {
// failure
console.log('failure', err);
});
models.Value.findOne({title:title}).exec() also returns a promise, so there is no need for callbacks or any additional casting here.
And if you don't normally use promises in your code, here is callback version of it:
function saveNewValue(id, title, callback) {
models.Value.create({
id:id,
title:title //this is a unique value
}).then(null, function(err) {
if (err.code === 11000) {
return models.Value.findOne({title:title}).exec()
} else {
throw err;
}
}).onResolve(callback);
}
Previous versions of Mongoose
If you're using any Mongoose version prior to 3.8.0, then you may need some help from when module:
var when = require('when'),
nodefn = require('when/node/function');
function saveNewValue(id, title) {
var thisValue = new models.Value({
id:id,
title:title //this is a unique value
});
var promise = nodefn.call(thisValue.save.bind(thisValue));
return promise.spread(function(product, numAffected) {
return product;
}).otherwise(function(err) {
if (err.code === 11000) {
return models.Value.findOne({title:title}).exec()
} else {
throw err;
}
});
}
I'm using nodefn.call helper function to turn callback-styled .save() method into a promise. Mongoose team promised to add promises support to it in Mongoose 4.x.
Then I'm using .spread helper method to extract the first argument from .save() callback.
I have this simple nodejs application, which generates dummy date for my web application.
All it does is:
Drops the dummy database
Populates the inventory collection
Populates the invoices collection
Populates the const data collection
Of course, all the actions are asynchronous and I want to execute them sequentially, one after another. For me, it was simpler to write something to manage this kind of the flow, however, I would like a mainstream solution, which would support other kinds of flows. For instance, running in parallel and stopping all on the first failure.
For your reference, please, find below the skeleton, depicting my solution:
/*global require, console, process*/
var mongo, db, inventory, createChain;
function generateInventory(count) {
// returns the generated inventory
}
function generateInvoices(count, inventory) {
// returns the generated invoices
}
function generateConst() {
// returns the generated const data
}
mongo = require('mongojs');
db = mongo.connect('dummy', ['invoices', 'const', 'inventory']);
createChain = function () {
"use strict";
var chain = [false], i = 0;
return {
add: function (action, errMsg, resultCallback) {
chain[chain.length - 1] = {action: action, errMsg: errMsg, resultCallback: resultCallback};
chain.push(false);
return this;
},
invoke: function (exit) {
var str, that = this;
if (chain[i]) {
chain[i].action(function (err, o) {
if (err || !o) {
str = chain[i].errMsg;
if (err && err.message) {
str = str + ": " + err.message;
}
console.log(str);
} else {
if (chain[i].resultCallback) {
chain[i].resultCallback(o);
}
i += 1;
that.invoke(exit);
}
});
} else {
console.log("done.");
if (exit) {
process.exit();
}
}
}
};
};
createChain()
.add(function (callback) {
"use strict";
console.log("Dropping the dummy database.");
db.dropDatabase(callback);
}, "Failed to drop the dummy database")
.add(function (callback) {
"use strict";
console.log("Populating the inventory.");
db.inventory.insert(generateInventory(100), callback);
}, "Failed to populate the inventory collection", function (res) {
"use strict";
inventory = res;
})
.add(function (callback) {
"use strict";
console.log("Populating the invoices.");
db.invoices.insert(generateInvoices(10, inventory), callback);
}, "Failed to populate the invoices collection")
.add(function (callback) {
"use strict";
console.log("Populating the const.");
db["const"].insert(generateConst(), callback);
}, "Failed to populate the const collection")
.invoke(true);
Can anyone suggest a relevant nodejs package, which would also be easy to use?
Thank you very much.
Use the async module to provide just about any type of flow control you're ever likely to need. In particular, the series method provides sequential flow control.
Actually, for sequential flow control, you should use waterfall
As an example:
async.waterfall([
function(cb){
cb(null,1);
},
function(r,cb){
// r=1
cb(null,2)
},
function(r,cb){
// r=2
cb(null,3)
}
],function(e,r){
// e=null
// r=3
})
This will execute sequentially.
If you callback an error early, (i.e. cb("error")), then it will directly go to the final function(e,r), with e="error" and r=undefined
Notice how function(r,cb){} can become precomposed in a util library to handle commonly reused blocks and make things in the future easier.