code sequence in nodejs - node.js

I am new to node.js and I thought I was beginning to understand asynchronous functions, but this code made me think that I did not understand it correctly anyway.
I am preparing for an insert to mongoDB with mongoose, and the object to insert is post. What made me wonder is that not always post.kunde or post.leverandor is set before the insert.
I thought that as long there is no asynchronous function, the code should execute line by line.
function create_post(account, dato, fakturanummer, bilag, bilagstype, supplier, customer,
descr, moms, amount, saldo, comId ) {
return new Promise(function(resolve, reject) {
var post = new Posteringer;
post.konto = account;
post.dato = dato;
post.fakturanummer = fakturanummer;
post.bilag = bilag;
post.bilagstype = bilagstype;
if (Object.keys(supplier).length) {
post.leverandor = supplier;
}
if (Object.keys(customer).length) {
post.kunde = customer;
}
post.tekst = descr;
post.moms = moms;
post.belob = amount;
post.saldo = saldo;
post.companyId = comId;
var promise = Accounts.findSingle(comId, account).exec();
promise.then(function(acc) {
post.navn = acc.Navn;
console.log(post);
post.save(function(err) {
if (err) {console.log(err.message);}
resolve(true);
});
});
});
}

So there are a few things wrong here. A promise is basically saying "hey, I'm not done doing my stuff yet, but I'll promise to get this to you when I figure out all the things I need to do."
I am not entirely sure how you are calling this, but think of this like a big callback function. So after this you'd have something like,
create_post(......).then(
function(post){
post.save(function(err){
if (err) {console.log(err.message);}
});
});
The real issue I see is you are only resolving one promise. I am not 100% sure if you have to resolve the Mongo promise, unless you are using bluebird promises with it. Your first promise never gets returned though.
I will redo your code, I think this should work:
function create_post(account, dato, fakturanummer, bilag, bilagstype, supplier, customer,
descr, moms, amount, saldo, comId ) {
return new Promise(function(resolve, reject) {
var post = new Posteringer;
post.konto = account;
post.dato = dato;
post.fakturanummer = fakturanummer;
post.bilag = bilag;
post.bilagstype = bilagstype;
if (Object.keys(supplier).length) {
post.leverandor = supplier;
}
if (Object.keys(customer).length) {
post.kunde = customer;
}
post.tekst = descr;
post.moms = moms;
post.belob = amount;
post.saldo = saldo;
post.companyId = comId;
var promise = Accounts.findSingle(comId, account).exec();
promise.then(function(acc) {
post.navn = acc.Navn;
console.log(post);
});
return (err ? reject(err) : resolve(post));
});
}
Then when you call this function call it like in my first example!
create_post(......).then(
function(post){
post.save(function(err){
if (err) {console.log(err.message);}
});
});
Literally what you are saying is if this resolves and I get my data without any errors, send back post. Then pass post into my callback function and do whatever you want to do with it.
Edit:
Make sure you always return something from your promises, as far as I know the async call will never resolve as it never receives anything. Although someone might have an example where this isn't true.

Related

combining results from several async/await calls (again)

Note to would-be closers or markers-as-duplicate : this question is not answered in How do I convert an existing callback API to promises?, as all answers there treat of calls in isolation and do not explain how to deal with successive, dependent calls.
This question is basically the same as in combining results from several async/await calls, but because of my slightly different context I fail to see how the solution therein can be adapted/mimicked.
I have two successive calls to a database, using an old API which only knows about callbacks. The second call needs objects/values returned by the first.
I have a working callback-version of the code, as follows :
connection.query(sql1,function(error1,results1,fields1) {
if(error1) {
console.log("Error during first query",error1);
}
let sql2=computeUsingFirstResult(result1);
connection.query(sql2,function(error2,results2,fields2) {
if(error2) {
console.log("Error during second query",error2);
}
doSomething(connection,results1,results2);
})
});
Here is my unsuccessful attempt to do it in async/await-style :
const util = require('util');
async function firstQuery(connection) {
return util.promisify(connection.query).call(sql1).catch(
error1 => console.log("Error during first query : ", error1)
);;
}
async function secondQuery(connection, result1) {
let sql2 = computeUsingFirstResult(result1);
return util.promisify(connection.query).call(sql2).catch(
error2 => console.log("Error during second query : ", error2)
);
}
let finalResult = {};
async function main() {
const results1 = await firstQuery(connection);
const results2 = await secondQuery(connection, results1);
doSomething(connection, results1, results2);
finalResult = [results1,results2];
console.log("Here is the finalResult : ", finalResult);
}
main().catch(
err => console.log("Something went wrong towards the end", err)
);
My async/await version fails as all the intermediate results are undefined. Yet, as far as I can see it should be equivalent to the non-async version above.
What is the correct way to do this ?
There are 2 approaches I am hoping you can try and see if either of them help:
(1) Bind the util.promisify to connection for both firstQuery and secondQuery something like so:
async function firstQuery(connection) {
return util.promisify(connection.query).bind(connection,sql1)().catch(
error1 => console.log("Error during first query : ", error1)
);;
}
// bind makes sure that internal bindings of connection object are in tact
If the above approach doesn't yield any result, try the next approach:
(2) Try using Promises-Wrapper instead of util.promisify for both firstQuery and secondQuery like so:
function firstQuery(connection,sql1){
return new Promise((resolve,reject)=>{
connection.query(sql1,function(error1,results1,fields1){
return error1 ? reject(error1):resolve(results1);
})
});
}
And now call them as you were in your code using Async/Await in the main function.
Few Potential bugs/typos I noticed in the code:
(1) firstQuery doesn't seem to be passed sql1 as one of its arguments, maybe this is intentional if sql1 exists in global scope.
(2) If you attach a catch block to both your queries, they will have an impact on your 2 calls (see comments in the code below):
async function main() {
const results1 = await firstQuery(connection); //<-- if this call fails, results1 will be undefined and the call will go to next step
const results2 = await secondQuery(connection, results1); // same thing here
doSomething(connection, results1, results2);
finalResult = [results1,results2];
console.log("Here is the finalResult : ", finalResult);
}
A possible solution is to remove the catch blocks from individual functions and just return the respective promises.
Then you can deal with them via a single try/catch block like this:
async function main() {
try{
const results1 = await firstQuery(connection);
const results2 = await secondQuery(connection, results1);
doSomething(connection, results1, results2);
finalResult = [results1,results2];
console.log("Here is the finalResult : ", finalResult);
}catch(err){
console.log("Error",err);
}
}
This will ensure that as soon as the first call fails, function goes straight to catch block without executing any other lines.

How do I achieve a synchronous requirement using asynchronous NodeJS

I am adding user validation an data modification page on a node.js application.
In a synchronous universe, in a single function I would:
Lookup the original record in the database
Lookup the user in LDAP to see if they are the owner or admin
Do the logic and write the record.
In an asynchronous universe that won't work. To solve it I've built a series of hand-off functions:
router.post('/writeRecord', jsonParser, function(req, res) {
post = req.post;
var smdb = new AWS.DynamoDB.DocumentClient();
var params = { ... }
smdb.query(params, function(err,data){
if( err == null ) writeRecordStep2(post,data);
}
});
function writeRecord2( ru, post, data ){
var conn = new LDAP();
conn.search(
'ou=groups,o=amazon.com',
{ ... },
function(err,resp){
if( err == null ){
writeRecordStep3( ru, post, data, ldap1 )
}
}
}
function writeRecord3( ru, post, data ){
var conn = new LDAP();
conn.search(
'ou=groups,o=amazon.com',
{ ... },
function(err,resp){
if( err == null ){
writeRecordStep4( ru, post, data, ldap1, ldap2 )
}
}
}
function writeRecordStep4( ru, post, data, ldap1, ldap2 ){
// Do stuff with collected data
}
Additionally, because the LDAP and Dynamo logic are in their own source documents, these functions are scattered tragically around the code.
This strikes me as inefficient, as well as inelegant. I'm eager to find a more natural asynchronous pattern to achieve the same result.
Any promise library should sort your issue out. My preferred choice is bluebird. In summary they help you in performing blocking operations.
If you haven't heard about bluebird then just use it. It converts all function of a module and return promise which is then-able. Simply put, it promisifies all functions.
Here is the mechanism:
Module1.someFunction() \\do your job and finally pass the return object to next call
.then() \\Use that object which is return from the first call, do your job and return the updated value
.then() \\same goes on
.catch() \\do your job when any error occurs.
Hope you understand. Here is an example:
var readFile = Promise.promisify(require("fs").readFile);
readFile("myfile.js",
"utf8").then(function(contents) {
return eval(contents);
}).then(function(result) {
console.log("The result of evaluating
myfile.js", result);
}).catch(SyntaxError, function(e) {
console.log("File had syntax error", e);
//Catch any other error
}).catch(function(e) {
console.log("Error reading file", e);
});
I could not tell from your pseudo-code exactly which async operations depend upon results from with other ones and knowing that is key to the most efficient way to code a series of asynchronous operations. If two operations do not depend upon one another, they can run in parallel which generally gets to an end result faster. I also can't tell exactly what data needs to be passed on to later parts of the async requests (too much pseudo-code and not enough real code to show us what you're really attempting to do).
So, without that level of detail, I'll show you two ways to approach this. The first runs each operation sequentially. Run the first async operation, when it's done, run the next one and accumulates all the results into an object that is passed along to the next link in the chain. This is general purpose since all async operations have access to all the prior results.
This makes use of promises built into the AWS.DynamboDB interface and makes our own promise for conn.search() (though if I knew more about that interface, it may already have a promise interface).
Here's the sequential version:
// promisify the search method
const util = require('util');
LDAP.prototype.searchAsync = util.promisify(LDAP.prototype.search);
// utility function that does a search and adds the result to the object passed in
// returns a promise that resolves to the object
function ldapSearch(data, key) {
var conn = new LDAP();
return conn.searchAsync('ou=groups,o=amazon.com', { ... }).then(results => {
// put our results onto the passed in object
data[key] = results;
// resolve with the original object (so we can collect data here in a promise chain)
return data;
});
}
router.post('/writeRecord', jsonParser, function(req, res) {
let post = req.post;
let smdb = new AWS.DynamoDB.DocumentClient();
let params = { ... }
// The latest AWS interface gets a promise with the .promise() method
smdb.query(params).promise().then(dbresult => {
return ldapSearch({post, dbresult}, "ldap1");
}).then(result => {
// result.dbresult
// result.ldap1
return ldapSearch(result, "ldap2")
}).then(result => {
// result.dbresult
// result.ldap1
// result.ldap2
// doSomething with all the collected data here
}).catch(err => {
console.log(err);
res.status(500).send("Internal Error");
});
});
And, here's a parallel version that runs all three async operations at once and then waits for all three of the to be done and then has all the results at once:
// if the three async operations you show can be done in parallel
// first promisify things
const util = require('util');
LDAP.prototype.searchAsync = util.promisify(LDAP.prototype.search);
function ldapSearch(params) {
var conn = new LDAP();
return conn.searchAsync('ou=groups,o=amazon.com', { ... });
}
router.post('/writeRecord', jsonParser, function(req, res) {
let post = req.post;
let smdb = new AWS.DynamoDB.DocumentClient();
let params = { ... }
Promise.all([
ldapSearch(...),
ldapSearch(...),
smdb.query(params).promise()
]).then(([ldap1Result, ldap2Result, queryResult]) => {
// process ldap1Result, ldap2Result and queryResult here
}).catch(err => {
console.log(err);
res.status(500).send("Internal Error");
});
});
Keep in mind that due to the pseudo-code nature of the code in your question, this is also pseudo-code where implementation details (exactly what parameters you're searching for, what response you're sending, etc...) have to be filled in. This should be illustrative of promise chaining to serialize operations and the use of Promise.all() for parallelizing operations and promisifying a method that didn't have promises built in.

Mongoose promise built in but not working?

Or quite possibly I am doing it wrong, in fact, more than likely I am doing it wrong.
Have a table which contains a "tree" of skill, starting at the root level and may be as deep as ten levels (only two so far), but I want to return it as one big fat JSON structure, so I want to ask the database for each set of data, build my structure then ask for the next level.
Of course if I just send of my requests using mongoose, they will come back at any time, as they are all nice asyncronous calls. Normally a good things.
Looking at the documentation for Mongoose(using 4.1.1) it seems like it has a promise built in, but whenever I try to use it the api call throws a hissy fit and I get a 500 back.
Here is my simple function:
exports.getSkills = function(req,res) {
console.log("Will return tree of all skills");
for (var i = 0; i<10; i++){
var returnData = [];
console.log("Lets get level " + i );
var query = Skill.find({level: i });//The query function
var promise = query.exec; //The promise?
promise.then(function(doc) { //Totally blows up at this point
console.log("Something came back")
return "OK";
});
}
}
The Mongoose documentation on the subject can be found here
http://mongoosejs.com/docs/api.html#promise_Promise
var promise = query.exec;
// =>
var promise = query.exec()
exports.getSkills = function(req,res) {
console.log("Will return tree of all skills");
var p;
for (var i = 0; i < 10; i ++) {
if (i == 0 ) {
p = Skill.find({level:i}).exec();
} else {
p.then(function (){
return Skill.find({level:i}).exec()
})
}
p.then(function (data) {
//deal with your data
})
}
p.then(function () {
// deal with response
})
}

Nodejs: Resolving promises with generator function

I know there are a lot of good examples over the web and I read a lot of them, but currently I'm stucked with resolving promises with the new functionality of generators in nodejs 0.11.x.
For e.g. I have the following function:
SolrBaseDomain.prototype.promisedQuery = function(query, callback) {
var solrClient = solr.createClient(this.configuration);
var defer = Q.defer();
solrClient.search(query, function(err,obj){
if (!err) {
if (obj.response.numFound > 0) {
defer.resolve(obj.response.docs);
} else {
defer.resolve(null);
}
} else {
defer.reject(err);
}
});
var promise = defer.promise;
return Q.async(function* (){
var result = yield promise;
return result;
});
};
I expected that every call to this method will wait until the promise is fullfilled and the return-statement gives back the result of the promise.
But currently it seems that instead the code inside "Q.async..." will not be executed or the async call arrives after the return statement of the method was executed.
It's strange, in every example I know, this is one of the recommended ways in order to wait for async calls in nodejs but currently it does not work for me.
I've tried a lot of different variations of the above example, but the result is everytime the same, I get not back a valid result.
I have nodejs installed in version 0.11.10 and the --harmony-flag is set, when the code is executede.
Can anyone point me to right direction? I'm wondering if I oversee something ... :)
Thanks for your feedback.
Best regards
Udo
I expected that every call to this method will wait until the promise is fullfilled and the return-statement gives back the result of the promise.
No. Generators will not make functions synchronous, you cannot (and don't want to) block while waiting for a result. When calling a generator function and running sequentially through the async steps that it yields, the result you will get back in the end is still asynchronous - and therefore a promise. Only inside of the generator, your code can use synchronous control flow and yield.
This means that the (then-) callback-based code
SolrBaseDomain.prototype.promisedQuery = function(query) {
var promise = Q.ninvoke(solr.createClient(this.configuration), "search", query);
return promise.then(function(obj) {
if (obj.response.numFound > 0) {
return obj.response.docs;
} else {
return null;
}
});
};
becomes
SolrBaseDomain.prototype.promisedQuery = Q.async(function* (query) {
var promise = Q.ninvoke(solr.createClient(this.configuration), "search", query);
var obj = yield promise;
// ^^^^^
if (obj.response.numFound > 0) {
return obj.response.docs;
} else {
return null;
}
});
Try this
SolrBaseDomain.prototype.promisedQuery = Q.async(function*(query) {
var solrClient = solr.createClient(this.configuration);
var obj = yield Q.ninvoke(solrClient, "search", query);
return obj.response.numFound > 0 ? obj.response.docs : null;
});
This does the same thing for promises as this does for callbacks:
SolrBaseDomain.prototype.query = function (query, callback) {
var solrClient = solr.createClient(this.configuration);
solrClient.search(query, function(err, obj) {
if (err) return callback(err);
callback(null, obj.response.numFound > 0 ? obj.response.docs : null);
});
};
Therefore if the first return a promise that resolves to undefined so will the callback version call the callback with undefined.
according to your suggestions, my code looks now like this:
...
SolrBaseDomain.prototype.query = Q.async(function* (query) {
var solrClient = solr.createClient(this.configuration);
var obj = yield Q.ninvoke(solrClient, "search", query);
return obj.response.numFound > 0 ? obj.response.docs : null;
});
...
I share the above query-function over all data access layers in order to have a central method which is querying the different indexes in an asynchronous way.
For e.g. in the domain data access layer, the code which deals with that function looks like this:
SolrHostDomain.prototype.getByName = Q.async(function* (domain) {
var queryObject = {
"domain": domain
};
var query = this.getQuery("byName", queryObject);
var docs = yield this.query(query);
var domain = null;
if (docs != null && docs.length > 0) {
domain = this.dataMapper.merge(docs[0]);
}
return domain;});
Currently I'm not sure if the generator in the "getByName"-function is necessary, but it seems to work. Dealing with promises is some unclear concept for me, since I'm new to nodejs.
So maybe, if you can help me on that topic and point me in the right direction, this would be helpfull.
The main question for me is, how can I ensure, that a synchronous method can call an asynchronous method and get back not a promise but the final result of this promise.
I've searched a long time, but I could not find a good documentation which describes the use of generator functions or promises in conjunction with synchronous calls. Even examples are focusing only of using the mechanism but not working together with synchronous function.
Best regards and many thanks for your help
Udo
Got it!!!
After a few trial and errors, I think I got it now and I have a working solutions:
Query function:
SolrBaseDomain.prototype.query = Q.async(function* (query) {
var solrClient = solr.createClient(this.configuration);
var obj = yield Q.ninvoke(solrClient, "search", query);
return obj.response.numFound > 0 ? obj.response.docs : null;
});
Calling method:
SolrHostDomain.prototype.getByName = function(domain) {
var queryObject = {
"domain": domain
};
var query = this.getQuery("byName", queryObject);
var docsPromise = this.query(query);
var _self = this;
return docsPromise.then(function(docs) {
var domain = null;
if (docs != null && docs.length > 0) {
domain = _self.dataMapper.merge(docs[0]);
}
return domain;
});
};
The solution was to understand, that the "query"-method still returns a promise instead of the concrete result even if yield is used.
So I have to add every code which is working on the result of the promise within the "then"-functions (or "done" if no other caller up in the calling hierarchy of methods will follow).
After the settlement of the promise, each code which is set within the "then"-functions will be processed.
BR
Udo

Node.js promises with mongoskin

I'm trying to avoid using callbacks when making mongodb queries. I'm using mongoskin to make calls like so:
req.db.collection('users').find().toArray(function (err, doc) {
res.json(doc);
});
In many cases I need to make multiple queries so I want to use Node.js promise library but I'm not sure how to wrap these functions as promises. Most of the examples I see are trivial for things like readFile, I'm guessing in this case I would need to wrap toArray somehow? Can this be done or would have to be something implemented by mongoskin?
An example could be any set of callbacks, find/insert, find/find/insert, find/update:
req.db.collection('users').find().toArray(function (err, doc) {
if (doc) {
req.db.collection('users').find().toArray(function (err, doc) {
// etc...
});
}
else {
// err
}
});
You can promisify the entire module like so with bluebird:
var Promise = require("bluebird");
var mongoskin = require("mongoskin");
Object.keys(mongoskin).forEach(function(key) {
var value = mongoskin[key];
if (typeof value === "function") {
Promise.promisifyAll(value);
Promise.promisifyAll(value.prototype);
}
});
Promise.promisifyAll(mongoskin);
This only needs to be done in one place for one time in your application, not anywhere in your application code.
After that you just use methods normally except with the Async suffix and don't pass callbacks:
req.db.collection('users').find().toArrayAsync()
.then(function(doc) {
if (doc) {
return req.db.collection('users').find().toArrayAsync();
}
})
.then(function(doc) {
if (doc) {
return req.db.collection('users').find().toArrayAsync();
}
})
.then(function(doc) {
if (doc) {
return req.db.collection('users').find().toArrayAsync();
}
});
So again, if you call a function like
foo(a, b, c, function(err, result) {
if (err) return console.log(err);
//Code
});
The promise-returning version is called like:
fooAsync(a, b, c).then(...)
(Uncaught errors are automatically logged so you don't need to check for them if you are only going to log it)
Just stumbled here with the same question and didn't love "promisfying" mongoskin so did a bit more digging and found monk. It's built on top of mongoskin, tidies up the API and returns
promises for all async calls. Probably worth a peek to anyone else who lands here.
Esailija's answer may work, but its not super efficient since you have to run db.collection on every single db call. I don't know exactly how expensive that is, but looking at the code in mongoskin, its non-trivial. Not only that, but it's globally modifying prototypes, which isn't very safe.
The way I do this with fibers futures is:
wrap the collection methods for each collection
on receiving the result, for methods that return a Cursor wrap the toArray method, call it and return the resulting future (for methods that don't return a cursor, you don't need to do anything else).
use the future as normal
like this:
var Future = require("fibers/future")
// note: when i originally wrote this answer fibers/futures didn't have a good/intuitive wrapping function; but as of 2014-08-18, it does have one
function futureWrap() {
// function
if(arguments.length === 1) {
var fn = arguments[0]
var object = undefined
// object, methodName
} else {
var object = arguments[0]
var fn = object[arguments[1]]
}
return function() {
var args = Array.prototype.slice.call(arguments)
var future = new Future
args.push(future.resolver())
var me = this
if(object) me = object
fn.apply(me, args)
return future
}
}
var methodsYouWantToHave = ['findOne', 'find', 'update', 'insert', 'remove', 'findAndModify']
var methods = {}
methodsYouWantToHave.forEach(function(method) {
internalMethods[method] = futureWrap(this.collection, method)
}.bind(this))
// use them
var document = methods.findOne({_id: 'a3jf938fj98j'}, {}).wait()
var documents = futureWrap(methods.find({x: 'whatever'}, {}).wait(), 'toArray')().wait()
If you don't want to use fibers, I'd recommend using the async-future module, which has a good wrap function built in too.

Resources