Proper way to make callbacks async by wrapping them using `co`? - node.js

It is 2016, Node has had nearly full ES6 support since v4, and Promises have been around since 0.12. It's time to leave callbacks in the dust IMO.
I'm working on a commander.js-based CLI util which leverages a lot of async operations - http requests and user input. I want to wrap the Commander actions in async functions so that they can be treated as promises, and also to support generators (useful for the co-prompt library I'm using for user input).
I've tried wrapping the CB with co in two ways:
1)
program.command('myCmd')
.action(program => co(function* (program) {...})
.catch(err => console.log(err.stack)) );
and
2) program.command('myCmd').action(co.wrap(function* (program) { .. }));
The problem with 1) is that the program parameter isn't passed
The problem with 2) is that errors are swallowed...
I'd really like to get this working as it yields much nicer code in my use case - involving a lot of http requests and also waiting for user input using the co-prompt library..
Is it a better option altogether perhaps to wrap program.Command.prototype.action somehow?
thanks!

I've used a bespoke version of something like co to get a db.exec function which uses yield to do database request. You can pass parameters into a generator function (I pass in a connection object - see the comment where I do it).
Here is by db.exec function that is very similar to what co does
exec(generator) {
var self = this;
var it;
debug('In db.exec iterator');
return new Promise((accept,reject) => {
debug('In db.exec Promise');
var myConnection;
var onResult = lastPromiseResult => {
debug('In db.exec onResult');
var obj = it.next(lastPromiseResult);
if (!obj.done) {
debug('db.exec Iterator NOT done yet');
obj.value.then(onResult,reject);
} else {
if (myConnection) {
myConnection.release();
debug('db.exec released connection');
}
accept(obj.value);
debug('db.exec Promise Resolved with value %d',obj.value);
}
};
self._connection().then(connection => {
debug('db.exec got a connection');
myConnection = connection;
it = generator(connection); //This passes it into the generator
onResult(); //starts the generator
}).catch(error => {
logger('database', 'Exec Function Error: ' + error.message);
reject(error);
});
});
}
the connection object also wraps by database connection object and provides a generator function ability to process the rows of the results from the database, but I won't post that here (although the example below is using it to process the rows).
Here is an example of using the exec function to run a sequence of sql
db.exec(function*(connection) {
if (params.name === ADMIN_USER) {
debug('Admin Logon');
user.name = ADMIN_DISPLAY;
user.keys = 'A';
user.uid = 0;
let sql = 'SELECT passwordsalt FROM Admin WHERE AdminID = 0';
connection.request(sql);
yield connection.execSql(function*() {
let row = yield;
if (row) {
user.nopass = (row[0].value === null);
} else {
user.nopass = false;
}
debug('Admin Password bypass ' + user.nopass.toString());
});
} else {
debug('Normal User Logon');
let sql = `SELECT u.UserID,PasswordSalt,DisplayName,AccessKey,l.LogID FROM Users u
LEFT JOIN UserLog l ON u.userID = l.userID AND DATEDIFF(D,l.LogDate,GETDATE()) = 0
WHERE u.UserName = #username`;
let request = connection.request(sql);
request.addParameter('username',db.TYPES.NVarChar,params.name);
let count = yield connection.execSql(function*() {
let row = yield;
if (row) {
user.uid = row[0].value;
user.name = row[2].value;
user.keys = (row[3].value === null) ? '' : row[3].value;
user.nopass = (row[1].value === null) ;
user.lid = (row[4].value === null) ? 0 : row[4].value;
debug('Found User with uid = %d and lid = %d, keys = %s',
user.uid, user.lid, user.keys);
}
});
if (count === 0) {
debug('Not Found User');
// couldn't find name in database
reply(false,false);
return;
}
}
if (!user.nopass) {
debug('Need a Password');
//user has a password so we must check it
passGood = false; //assume false as we go into this
let request = connection.request('CheckPassword');
request.addParameter('UserID',db.TYPES.Int,user.uid);
request.addParameter('password',db.TYPES.VarChar,params.password);
yield connection.callProcedure(function*() {
let row = yield;
if (row) {
//got a valid row means we have a valid password
passGood = true;
}
});
} else {
passGood = true;
}
if (!passGood) {
debug('Not a Good Pasword');
reply(false,true);
} else {
if (user.uid !== 0 && user.lid === 0) {
let sql = `INSERT INTO UserLog(UserID,LogDate,TimeOn,UserName) OUTPUT INSERTED.logID
VALUES(#uid,GETDATE(),GETDATE(),#username)`;
let request = connection.request(sql);
request.addParameter('uid',db.TYPES.Int,user.uid);
request.addParameter('username',db.TYPES.NVarChar,user.name);
yield connection.execSql(function*() {
let row = yield;
if (row) {
user.lid = row[0].value;
debug('Users Log Entry = %d',user.lid);
}
});
}
reply(true,user);
}
})
.catch((err) => {
logger('database','Error on logon: ' + err.message);
reply(false,false);
});
});

There is a quite simple way to do async function in Commander.js
async function run() {
/* code goes here */
}
program
.command('gettime')
.action(run);
program.parse(process.argv);

Related

Resolving a recursive socket.on("data") call

I have a node socket server that needs to receive data more than once during a single socket.on('data') call. It's sorta like socket.on('data') is running recursively. I think I've just configured this really really wrong, but I don't know enough NodeJS to figure it out.
Here's an example of the issue:
const net = require("net");
const socket = new net.Socket();
const port = process.argv[2];
socket.connect(port, "127.0.0.1", () => {
// We communicate with another server, always with "type" -> "result" messages.
socket.write(JSON.stringify({ type: "connect", result: "JavaScript" }));
});
function get_query(query) {
queryMessage = JSON.stringify({
type: "query",
result: query,
});
socket.write(queryMessage);
// This is the critical line that I don't know how to resolve.
// Right at this point I need wait for another socket.on('data') call
return socket.giveMeThatQuery()
}
socket.on("data", (data) => {
let jsonData = JSON.parse(data);
if (jsonData["type"] == "call_func") {
let funcToCall = func_reg[jsonData["result"]];
// This will need to call get_query 0-n times, dynamically
result = funcToCall();
let finishMessage = JSON.stringify({
type: "finish_func",
result,
});
socket.write(finishMessage);
} else if (jsonData["type"] == "query_response") {
// This has to connect back to get_query somehow?
// I really have no idea how to approach this.
// async functions? globals?
giveDataToQuery()
}
});
function func1() {
return "bird"; // some funcs are simple
}
function func2() {
// some are more complicated
let data = get_query("Need " + get_query("some data"));
return "cat" + data;
}
function func3() {
let data = get_query("Need a value");
let extra = '';
if (data == "not good") {
extra = get_query("Need more data");
}
return data + "dog" + extra;
}
var func_reg = { func1, func2, func3};
This server is just an example, I can edit or provide more context if it's needed. I suppose if you're looking at this question right after it's posted I can explain more on the live stream.
Edit: adding an example of using globals and while true (this is how I did it in Python). However, in Python, I also put the entire funcToCall execution on another thread, which it doesn't seem like is possible in Node. I'm beginning to think this is actually just impossible, and I need to refactor the entire project design.
let CALLBACK = null;
function get_query(query) {
queryMessage = JSON.stringify({
type: "query",
result: query,
});
socket.write(queryMessage);
while (true) {
if (CALLBACK) {
let temp = CALLBACK;
CALLBACK = null;
return temp;
}
}
...
} else if (jsonData["type"] == "query_response") {
CALLBACK = jsonData["result"];
}
I was correct, what I wanted to do was impossible. Node (generally) only has one thread, ever. You can only be waiting in one place at a time. Async threads or callbacks doesn't solve this, since the callback I would need is another socket.on, which makes no sense.
The solution is yield. This allows me to pause the execution of a function and resume at a later time:
function* func2() {
let data = (yield "Need " + (yield "some data"));
yield "cat" + data;
yield 0;
}
function* func3() {
let data = (yield "Need a value");
let extra = '';
if (data == "not good") {
extra = (yield "Need more data");
}
yield data + "dog" + extra;
yield 0;
}
I updated the server.on to handle these generator function callbacks and respond differently to the yield 0;, which indicates the function has no queries. Here's the final server.on.
server.on("data", (data) => {
try {
let jsonData = JSON.parse(data);
let queryResult= null;
let queryCode = null;
if (jsonData["type"] == "call_func") {
var script = require(jsonData["func_path"]);
FUNC_ITER = script.reserved_name();
} else if (jsonData["type"] == "queryResult") {
queryResult = jsonData["result"];
}
queryCode = FUNC_ITER.next(queryResult);
if (queryCode.value == 0) {
let finishMessage = JSON.stringify({
type: "finish_func",
result: null,
});
server.write(finishMessage);
} else {
let exeMessage = JSON.stringify({
type: "queryCode",
result: queryCode.value,
});
server.write(exeMessage);
}
} catch (err) {
crashMessage = JSON.stringify({
type: "crash",
result: err.stack,
});
server.write(crashMessage);
}
});
And an example of the node files I generated containing the custom functions:
function* reserved_name() {
(yield "return " + String("\"JavaScript concat: " + (yield "value;") + "\"") + ";")
yield 0;
}
module.exports = { reserved_name };
If you're wondering, this is for a language called dit, which can execute code in other languages. There's a Python server that talks to these other languages. Here's an example I finished last week which uses Python, Node, and Lua as the guest languages:

nodejs retry function if failed X times

I want my function to execute X(=3) times until success.
In my situation I'm running kinesis.putRecord (from AWS API), and if it fails - I want to run it again until it succeeds, but not more than 3 tries.
I'm new to NodeJS, and the code I wrote smells bad.
const putRecordsPromise = function(params){
return new Promise((resolve, reject) => {
kinesis.putRecord(params, function (err, data) {
resolve(err)
});
})
}
async function waterfall(params){
try{
let triesCounter = 0;
while(triesCounter < 2){
console.log(`try #${triesCounter}`)
let recordsAnswer = await putRecordsPromise(params)
if(!recordsAnswer){
console.log("success")
break;
}
triesCounter += 1;
}
// continue ...
} catch(err){
console.error(err)
}
}
waterfall(params)
I promise the err result. Afterwards, If the err is empty, then all good. otherwise, continue running the same command.
I'm sure there is a smarter way to do this. Any help would be appreciated.
I think, all the Aws functions can return a Promise out of the box, then you can just put the call into try/catch:
let triesCounter = 0;
while(triesCounter < 2){
console.log(`try #${triesCounter}`)
try {
await kinesis.putRecord(params).promise();
break; // 'return' would work here as well
} catch (err) {
console.log(err);
}
triesCounter ++;
}
In functional style:
...
await tryUntilSucceed(() => kinesis.putRecord(params).promise());
...
async function tryUntilSucceed(promiseFn, maxTries=3) {
try {
return await promiseFn();
} catch (e) {
if (maxTries > 0) {
return tryUntilSucceed(promiseFn, maxTries - 1);
}
throw e;
}
}
Make a little module, say try-and-try-again.js:
exports = module.exports = tryAndTryAgain;
function tryAndTryAgain( maxTries, thisContext , fn, ...argv) {
let success = false;
for (let i = i ; i < maxTries && !success ; ++i ) {
let rc = fn.apply(thisContext, args);
success = rc == 0 ? true : false;
}
return success;
}
Then you can use it anywhere:
const tryAndTryAgain = require('./try-and-try-again');
function somethingThatMightNeedARetry() { ... }
const succeeded = tryAndTryAgain( 3 , null, somethingThatMightNeedARetry, 'arg-1', 'arg-2', 'arg-3' );
There is an npm package called async-retry that is pretty handy. It acts as a wrapper for your function and retries if anything throws (with some exceptions that you can handle, see their example below).
// Packages
const retry = require('async-retry')
const fetch = require('node-fetch')
await retry(async bail => {
// if anything throws, we retry
const res = await fetch('https://google.com')
if (403 === res.status) {
// don't retry upon 403
bail(new Error('Unauthorized'))
return
}
const data = await res.text()
return data.substr(0, 500)
}, {
retries: 5
})

node wait for iteration to complete before callback

I have a lambda function in node.js to send a push notification.
In that function I need to iterate through my users sending a notification for each one prior to the callback.
Ideally I would like the iteration to perform in parallel.
What would be the best way to do this?
My code is currently as follows but it does not work as expected because the last user is not always the last to be handled:
var apnProvider = new apn.Provider(options);
var iterationComplete = false;
for (var j = 0; j < users.length; j++) {
if (j === (users.length - 1)) {
iterationComplete = true;
}
var deviceToken = users[j].user_device_token;
var deviceBadge = users[j].user_badge_count;
var notification = new apn.Notification();
notification.alert = message;
notification.contentAvailable = 1;
notification.topic = "com.example.Example";
apnProvider.send(notification, [deviceToken]).then((response) => {
if (iterationComplete) {
context.succeed(event);
}
});
}
Use Promise.all instead - map each user's associated apnProvider.send call to a Promise in an array, and when all Promises in the array are resolved, call the callback:
const apnProvider = new apn.Provider(options);
const userPromises = users.map((user) => {
const deviceToken = user.user_device_token;
const deviceBadge = user.user_badge_count;
const notification = new apn.Notification();
notification.alert = message;
notification.contentAvailable = 1;
notification.topic = "com.example.Example";
return apnProvider.send(notification, [deviceToken]);
})
Promise.all(userPromises)
.then(() => {
context.succeed(event);
})
.catch(() => {
// handle errors
});

Using Async/Await in WATSON Nodejs SDK

I am building a chatbot with WATSON API where I use the async/await method in order to fetch the data from MongoDB and attain the result, which then I send it back to the user.
The function artpromise is the promise that collects data from mongo DB. And the function randomartist is a function that fetches 3 random document from the DB. However, the WATSON BLUEMIX Cloud service supports Nodejs SDK of 6.1.3 which does not support the async method. Is there any way to update the SDK version on Blumix or should I use a difference approach in fetching data from the server?
let getConversationResponse = (message, context) => {
let payload = {
workspace_id: process.env.WORKSPACE_ID,
context: context || {},
input: message || {}
};
payload = preProcess(payload);
return new Promise((resolved, rejected) => {
// Send the input to the conversation service
conversation.message(payload, async function(err, data) {
if (err) {
rejected(err);
}
else{
if(data.context.type == 'ask'){
let artist = data.context.name;
let result = await artpromise(artist);
console.log(result);
data.context.name = result[0].name;
data.context.nationality = result[0].nationality;
data.context.birth = result[0].years;
data.context.url = result[0].art_link;
data.output.text = data.context.name+' is a '+data.context.nationality+' artist from '+data.context.birth+'. Check out a painting at '+data.context.url;
}
else if(data.context.type == 'random_artist'){
let result = await randomArtist();
console.log(result);
data.output.text = 'Let\'s find some random artists for you! \n'+result;
}
let processed = postProcess(data);
if(processed){
// return 값이 Promise 일 경우
if(typeof processed.then === 'function'){
processed.then(data => {
resolved(data);
}).catch(err => {
rejected(err);
})
}
// return 값이 변경된 data일 경우
else{
resolved(processed);
}
}
else{
// return 값이 없을 경우
resolved(data);
}
}
});
})
}
Using Node's util.promisify() utility, you can transform a callback-style function into a Promise-based one.
Somewhere outside of your getConversationResponse-function, assign it to a local variable:
const util = require('util');
const messagePromise = util.promisify(conversation.message);
And use that function instead. Something like this should work:
const util = require('util');
const messagePromise = util.promisify(conversation.message);
let getConversationResponse = async (message, context) => {
let payload = preprocess({
workspace_id: process.env.WORKSPACE_ID,
context: context || {},
input: message || {}
});
let data = await messagePromise(payload);
if (data.context.type == 'ask') {
let artist = data.context.name;
let result = await artpromise(artist);
console.log(result)
data.context.name = result[0].name;
data.context.nationality = result[0].nationality;
data.context.birth = result[0].years;
data.context.url = result[0].art_link;
data.output.text = data.context.name+' is a '+data.context.nationality+' artist from '+data.context.birth+'. Check out a painting at '+data.context.url;
} else if (data.context.type == 'random_artist'){
let result = await randomArtist();
console.log(result);
data.output.text = 'Let\'s find some random artists for you! \n'+result;
}
return postProcess(data) || data;
};
Note that if the return value of postProcess is falsy, it will return the data variable instead. Additionally, an async function always returns a Promise, so to call this function, you'll do:
getConversationResponse(message, context).then((data) => {
// Do something with the data
}).catch((e) => {
// Handle the error!
});
or if you call it from another async function:
let data = await getConversationResponse(message, context);
or if you need to specifically catch errors in the calling async function:
try {
let data = await getConversationResponse(message, context);
} catch (e) {
// Handle error
}
Just like regular synchronous code, any error thrown in the function call chain "trickles up" to the top-most callee. If you're confused about this, I suggest reading up on error handling.
If you want to use the Watson API in an async Promise-based fashion throughout your code, it might be feasible to write a small wrapper library and use that directly instead.
A Promise-only implementation:
const util = require('util');
const messagePromise = util.promisify(conversation.message);
let getConversationResponse = (message, context) => {
let payload = preprocess({
workspace_id: process.env.WORKSPACE_ID,
context: context || {},
input: message || {}
});
return messagePromise(payload).then((data) => {
if (data.context.type == 'ask') {
let artist = data.context.name;
return artpromise(artist).then((result) => {
data.context.name = result[0].name;
data.context.nationality = result[0].nationality;
data.context.birth = result[0].years;
data.context.url = result[0].art_link;
data.output.text = data.context.name+' is a '+data.context.nationality+' artist from '+data.context.birth+'. Check out a painting at '+data.context.url;
return data;
});
} else if (data.context.type == 'random_artist') {
return randomArtist().then((result) => {
data.output.text = 'Let\'s find some random artists for you! \n' + result;
return data;
});
}
}).then((data) => {
return postProcess(data) || data;
});
};
Calling it is the exact same as the async/await implementation.

How do I return a value inside a Promise? In "then()" and "catch()"

I just want to return 1 and return 0 at the specified places. I've looked at numerous sources but was unable to solve this issue.
Below is the code :
exports.getLatLng = function(row){
var attractionId = row['attractionid'];
var attractionName = row['attractionname'] + ' ' +
row['Destination_name'];
var googleMapsResults;
return googleMapsClient.geocode({address: attractionName}).asPromise()
.then((response) => {
googleMapsResults = response.json.results[0];
// console.log(googleMapsResults);
model.dumpIntoMongo(attractionId, googleMapsResults);
// var y=tmp[0];
var latitude = googleMapsResults.geometry.location.lat;
var longitude = googleMapsResults.geometry.location.lng;
row["id"] = parseInt(attractionId);
// delete row['attractionid'];
delete row['destination_id'];
delete row['destination_name'];
delete row['attraction_id'];
delete row['success'];
// row["lat"] = latitude;
// row["lon"] = longitude;
row["latlon"] = latitude.toString() + "," + longitude.toString();
exports.indexIntoSolr(row);
return 1; //return 1
})
.catch((err) => {
console.log(err);
return 0; // return 0
});
}
In case If you want another more clear implementation, you can refer this one.
Here, First write a function which returns some data as promise after certain calculations:
function getSomething(credentials) {
const {
name
} = credentials;
return new Promise((resolve, reject) => {
if(everything_as_expected){
return resolve(some_data)
}
else if(some_error){
return reject(error);
}
});
}
To handle results/Data returned by the function( promisely ), call that previous function:
getSomething(credentials)
.then((message) => {
console.log(`${message} got after success.`);
})
.catch((error_message) => {
console.log(error_message);
});
The method returns a promise that is pending until it resolves. To get a value you need to attach a continuation to it:
getLatLng(row).then( result => {
console.log( result );
});
This will definitely display 0 or 1 as your internal implementation of the getLatLng looks like it correctly handles both execution paths.

Resources