Generic wrapper to harmonise functions to async style - node.js

I would love to write a generic wrapper that takes a function, and returns the "async-style" version of that function IF it wasn't async to start with.
Trouble is, there is no easy way to know whether the call is sync or async. So... this basically "cannot be done". Right?
(Note that the wrapper should harmonise sync functions to async style, and LEAVE async functions alone)
var wrapper = function( fn ){
return function(){
var args = Array.prototype.splice.call(arguments, 0);
var cb = args[ args.length - 1 ];
// ?!?!?!?!?
// I cannot actually tell if `fn` is sync
// or async, and cannot determine it!
console.log( fn.toString() );
}
}
var f1Async = wrapper( function( arg, next ){
next( null, 'async' + arg );
})
var f2Sync = wrapper( function( arg ){
return 'sync' + arg;
})
f1Async( "some", function(err, ret ){
console.log( ret );
});
f2Sync( "some other", function(err, ret ){
console.log( ret );
});

You cannot find out what the accepted arguments of a function are, so you cannot find out if it takes a callback or not.

In javascript there is no way to check if the last argument of a function is a function, because in javascript you do not define the types of your arguments.
My solution works by getting a list of the parameters in the function, then using a RegExp to see if that parameter is used as a function. Also, in the case that the callback is not being used directly (like passing it to something else), it has a list of argument names to be considered as a callback.
And the code is:
var STRIP_COMMENTS = /((\/\/.*$)|(\/\*[\s\S]*?\*\/))/mg;
var CALLBACK_NAMES = [ "next", "done", "callback", "cb"];
function getParamNames(func) {
var fnStr = func.toString().replace(STRIP_COMMENTS, '')
var result = fnStr.slice(fnStr.indexOf('(')+1, fnStr.indexOf(')')).match(/([^\s,]+)/g)
if(result === null)
result = []
return result
}
function checkIfParamIsFunction(func, paramName){
var fnStr = func.toString();
if (fnStr.replace(new RegExp("(" + paramName + "\s*\([A-Za-z0-9,\.]*\)?!{|" + paramName + ".apply\([A-Za-z0-9,\.]*\)|" + paramName + ".call\([A-Za-z0-9,\.]*\))", ""), "{<._|/}") != fnStr) { // Remove All Calls to the arg as a function, then check to see if anything was changed.
return true;
} else {
return false;
}
}
function makeAsync(func) {
var paramNames = getParamNames(func);
if (checkIfParamIsFunction(func, paramNames[paramNames.length - 1])
|| CALLBACK_NAMES.indexOf(paramNames[paramNames.length - 1]) != -1) {
// Function Is Already async
return func;
} else {
return function () {
var args = Array.prototype.slice.call(arguments);
var cb = args.pop();
cb(func.apply(this, args));
}
}
}
function test1(a){
return (a+' test1');
};
function test2(a, callback){
return callback(a+' test2')
};
function main(){
var tested1 = makeAsync(test1);
var tested2 = makeAsync(test2);
tested1('hello', function(output){
console.log('holy shit it\'s now async!!');
console.log(output);
});
tested2('world', function(output){
console.log('this one was already async tho');
console.log(output);
});
}
main();
Simply call makeAsync(function) and it will return an async function. This will work if you use function.apply or .call.

It simply cannot be done. End of story.

Though, this is not the answer, but a good alternative. I have provided example of browser based JavaScript but same class can be used on Node as well.
To solve this problem, promises were developed. However we use a modified version of promise as follow.
function AtomPromise(f)
{
// Success callbacks
this.sc = [];
// Error callbacks
this.ec = [];
this.i = f;
this.state = 'ready';
}
AtomPromise.prototype ={
finish: function(s,r) {
this.result = r;
var c = s ? this.sc : this.ec;
this.state = s ? 'done' : 'failed' ;
for(var i=o ; i< c.length; i++){
c[i](this);
}
},
invoke: function(f) {
this.state = 'invoking';
this.i(this);
},
then: function(f) {
this.sc.push(f);
},
failed: function(f){
this.ec.push(f);
},
value: function(v) {
if(v !== undefined ) this.result = v;
return this.result;
},
pushValue: function(v) {
var _this = this;
setTimeout(100, function () {
_this.finish(true, v);
});
}
}
//wrap jQuery AJAX
AtomPromise.ajax = function( url, options ) {
return new AtomPromise(function (ap){
$.ajax(url, options)
.then( function(r){ ap.finish(true, r); })
.failed( function (){ ap.finish( false, arguments) });
}) ;
}
//Wrape sync function
AtomPromise.localStorage = function( key ) {
return new AtomPromise(function (ap){
var v = localStorage[key];
ap.pushValue(v);
}) ;
}
// Calling Sequence
AtomPromise.ajax( 'Url' ).then( function(ap) {
alert(ap.value());
}).invoke();
AtomPromise.localStorage( 'Url' ).then( function(ap) {
alert(ap.value());
}).invoke();
Both functions are now asynchronous. Push Value method makes result route through setTimeout that makes further calls asynchronous.
This is used in Web Atoms JS to wrape async code into single attribute and by following one pattern you can get rid of async callback hell. http://webatomsjs.neurospeech.com/docs/#page=concepts%2Fatom-promise.html
Disclaimer: I am author of Web Atoms JS.

Related

Resolving a recursive socket.on("data") call

I have a node socket server that needs to receive data more than once during a single socket.on('data') call. It's sorta like socket.on('data') is running recursively. I think I've just configured this really really wrong, but I don't know enough NodeJS to figure it out.
Here's an example of the issue:
const net = require("net");
const socket = new net.Socket();
const port = process.argv[2];
socket.connect(port, "127.0.0.1", () => {
// We communicate with another server, always with "type" -> "result" messages.
socket.write(JSON.stringify({ type: "connect", result: "JavaScript" }));
});
function get_query(query) {
queryMessage = JSON.stringify({
type: "query",
result: query,
});
socket.write(queryMessage);
// This is the critical line that I don't know how to resolve.
// Right at this point I need wait for another socket.on('data') call
return socket.giveMeThatQuery()
}
socket.on("data", (data) => {
let jsonData = JSON.parse(data);
if (jsonData["type"] == "call_func") {
let funcToCall = func_reg[jsonData["result"]];
// This will need to call get_query 0-n times, dynamically
result = funcToCall();
let finishMessage = JSON.stringify({
type: "finish_func",
result,
});
socket.write(finishMessage);
} else if (jsonData["type"] == "query_response") {
// This has to connect back to get_query somehow?
// I really have no idea how to approach this.
// async functions? globals?
giveDataToQuery()
}
});
function func1() {
return "bird"; // some funcs are simple
}
function func2() {
// some are more complicated
let data = get_query("Need " + get_query("some data"));
return "cat" + data;
}
function func3() {
let data = get_query("Need a value");
let extra = '';
if (data == "not good") {
extra = get_query("Need more data");
}
return data + "dog" + extra;
}
var func_reg = { func1, func2, func3};
This server is just an example, I can edit or provide more context if it's needed. I suppose if you're looking at this question right after it's posted I can explain more on the live stream.
Edit: adding an example of using globals and while true (this is how I did it in Python). However, in Python, I also put the entire funcToCall execution on another thread, which it doesn't seem like is possible in Node. I'm beginning to think this is actually just impossible, and I need to refactor the entire project design.
let CALLBACK = null;
function get_query(query) {
queryMessage = JSON.stringify({
type: "query",
result: query,
});
socket.write(queryMessage);
while (true) {
if (CALLBACK) {
let temp = CALLBACK;
CALLBACK = null;
return temp;
}
}
...
} else if (jsonData["type"] == "query_response") {
CALLBACK = jsonData["result"];
}
I was correct, what I wanted to do was impossible. Node (generally) only has one thread, ever. You can only be waiting in one place at a time. Async threads or callbacks doesn't solve this, since the callback I would need is another socket.on, which makes no sense.
The solution is yield. This allows me to pause the execution of a function and resume at a later time:
function* func2() {
let data = (yield "Need " + (yield "some data"));
yield "cat" + data;
yield 0;
}
function* func3() {
let data = (yield "Need a value");
let extra = '';
if (data == "not good") {
extra = (yield "Need more data");
}
yield data + "dog" + extra;
yield 0;
}
I updated the server.on to handle these generator function callbacks and respond differently to the yield 0;, which indicates the function has no queries. Here's the final server.on.
server.on("data", (data) => {
try {
let jsonData = JSON.parse(data);
let queryResult= null;
let queryCode = null;
if (jsonData["type"] == "call_func") {
var script = require(jsonData["func_path"]);
FUNC_ITER = script.reserved_name();
} else if (jsonData["type"] == "queryResult") {
queryResult = jsonData["result"];
}
queryCode = FUNC_ITER.next(queryResult);
if (queryCode.value == 0) {
let finishMessage = JSON.stringify({
type: "finish_func",
result: null,
});
server.write(finishMessage);
} else {
let exeMessage = JSON.stringify({
type: "queryCode",
result: queryCode.value,
});
server.write(exeMessage);
}
} catch (err) {
crashMessage = JSON.stringify({
type: "crash",
result: err.stack,
});
server.write(crashMessage);
}
});
And an example of the node files I generated containing the custom functions:
function* reserved_name() {
(yield "return " + String("\"JavaScript concat: " + (yield "value;") + "\"") + ";")
yield 0;
}
module.exports = { reserved_name };
If you're wondering, this is for a language called dit, which can execute code in other languages. There's a Python server that talks to these other languages. Here's an example I finished last week which uses Python, Node, and Lua as the guest languages:

Resolving a deferred Q multiple times

As I know deferred can be resolved only once and also nodejs caches the required module at the first step in order to prevent loading it every single time, but what I want to do is renewing deferred Q object each time I want to resolve/reject a returned value.
This is my server code :
// server.js
app.get('/api/posts', function(req,res,next){
var insta = require('./upRunner'); // this's the main obstacle
insta.returner().then(function(data){
// ....
};
});
and the code for ./upRunner.js is :
// upRunner.js
...
var defer = Q.defer();
upRunner();
function extractor(body) {
var $ = cheerio.load(body), scripts= [];
$('script').each(function(i, elem){
scripts[i] = $(this).text();
});
var str = scripts[6],
newData = JSON.parse(str.substring(str.indexOf("{"), str.lastIndexOf(";"))).entry_data.TagPage[0].tag.media.page_info;
grabber(str);
return newData;
}
function grabber(str) {
newData = JSON.parse(str.substring(str.indexOf("{"), str.lastIndexOf(";"))).entry_data.TagPage[0].tag.top_posts.nodes;
newData.sort(dynamicSort('-date'));
newData.forEach(function(elem,index,array){
if (instaImages.length >= 10) {
defer.resolve(instaImages);
} else {
instaImages.push(elem);
}
});
}
function upRunner(newData){
profilePage = !(tagPage = URL.includes("/tags/") ? true : false);
if (!newData) {
request(URL,{jar:true}, function(err, resp, body){
var $ = cheerio.load(body), scripts= [];
$('script').each(function(i, elem){
scripts[i] = $(this).text();
});
var str = scripts[6],
newData = JSON.parse(str.substring(str.indexOf("{"), str.lastIndexOf(";"))).entry_data.TagPage[0].tag.media.page_info;
upRunner(newData);
});
} else {
if (newData.has_next_page) {
requester(URL, newData.end_cursor).then(function(body){
newData = extractor(body);
upRunner(newData);
});
} else {
console.log('it\'s finished\n');
}
function returner() {
return deferred.promise;
}
exports.returner = returner;
As you see I'm almost renewing upRunner returner deferred promise each time server get /api/posts address, but the problem is the deferred object still return the old resolved value.
There is the grabber function which resolve value, so defer can not be localized in a single function.
Try to initialized deferred value locally to get new resolve value.
function returner() {
var defer= $q.defer();
return deferred.promise;
};
exports.returner = returner;

Proper way to make callbacks async by wrapping them using `co`?

It is 2016, Node has had nearly full ES6 support since v4, and Promises have been around since 0.12. It's time to leave callbacks in the dust IMO.
I'm working on a commander.js-based CLI util which leverages a lot of async operations - http requests and user input. I want to wrap the Commander actions in async functions so that they can be treated as promises, and also to support generators (useful for the co-prompt library I'm using for user input).
I've tried wrapping the CB with co in two ways:
1)
program.command('myCmd')
.action(program => co(function* (program) {...})
.catch(err => console.log(err.stack)) );
and
2) program.command('myCmd').action(co.wrap(function* (program) { .. }));
The problem with 1) is that the program parameter isn't passed
The problem with 2) is that errors are swallowed...
I'd really like to get this working as it yields much nicer code in my use case - involving a lot of http requests and also waiting for user input using the co-prompt library..
Is it a better option altogether perhaps to wrap program.Command.prototype.action somehow?
thanks!
I've used a bespoke version of something like co to get a db.exec function which uses yield to do database request. You can pass parameters into a generator function (I pass in a connection object - see the comment where I do it).
Here is by db.exec function that is very similar to what co does
exec(generator) {
var self = this;
var it;
debug('In db.exec iterator');
return new Promise((accept,reject) => {
debug('In db.exec Promise');
var myConnection;
var onResult = lastPromiseResult => {
debug('In db.exec onResult');
var obj = it.next(lastPromiseResult);
if (!obj.done) {
debug('db.exec Iterator NOT done yet');
obj.value.then(onResult,reject);
} else {
if (myConnection) {
myConnection.release();
debug('db.exec released connection');
}
accept(obj.value);
debug('db.exec Promise Resolved with value %d',obj.value);
}
};
self._connection().then(connection => {
debug('db.exec got a connection');
myConnection = connection;
it = generator(connection); //This passes it into the generator
onResult(); //starts the generator
}).catch(error => {
logger('database', 'Exec Function Error: ' + error.message);
reject(error);
});
});
}
the connection object also wraps by database connection object and provides a generator function ability to process the rows of the results from the database, but I won't post that here (although the example below is using it to process the rows).
Here is an example of using the exec function to run a sequence of sql
db.exec(function*(connection) {
if (params.name === ADMIN_USER) {
debug('Admin Logon');
user.name = ADMIN_DISPLAY;
user.keys = 'A';
user.uid = 0;
let sql = 'SELECT passwordsalt FROM Admin WHERE AdminID = 0';
connection.request(sql);
yield connection.execSql(function*() {
let row = yield;
if (row) {
user.nopass = (row[0].value === null);
} else {
user.nopass = false;
}
debug('Admin Password bypass ' + user.nopass.toString());
});
} else {
debug('Normal User Logon');
let sql = `SELECT u.UserID,PasswordSalt,DisplayName,AccessKey,l.LogID FROM Users u
LEFT JOIN UserLog l ON u.userID = l.userID AND DATEDIFF(D,l.LogDate,GETDATE()) = 0
WHERE u.UserName = #username`;
let request = connection.request(sql);
request.addParameter('username',db.TYPES.NVarChar,params.name);
let count = yield connection.execSql(function*() {
let row = yield;
if (row) {
user.uid = row[0].value;
user.name = row[2].value;
user.keys = (row[3].value === null) ? '' : row[3].value;
user.nopass = (row[1].value === null) ;
user.lid = (row[4].value === null) ? 0 : row[4].value;
debug('Found User with uid = %d and lid = %d, keys = %s',
user.uid, user.lid, user.keys);
}
});
if (count === 0) {
debug('Not Found User');
// couldn't find name in database
reply(false,false);
return;
}
}
if (!user.nopass) {
debug('Need a Password');
//user has a password so we must check it
passGood = false; //assume false as we go into this
let request = connection.request('CheckPassword');
request.addParameter('UserID',db.TYPES.Int,user.uid);
request.addParameter('password',db.TYPES.VarChar,params.password);
yield connection.callProcedure(function*() {
let row = yield;
if (row) {
//got a valid row means we have a valid password
passGood = true;
}
});
} else {
passGood = true;
}
if (!passGood) {
debug('Not a Good Pasword');
reply(false,true);
} else {
if (user.uid !== 0 && user.lid === 0) {
let sql = `INSERT INTO UserLog(UserID,LogDate,TimeOn,UserName) OUTPUT INSERTED.logID
VALUES(#uid,GETDATE(),GETDATE(),#username)`;
let request = connection.request(sql);
request.addParameter('uid',db.TYPES.Int,user.uid);
request.addParameter('username',db.TYPES.NVarChar,user.name);
yield connection.execSql(function*() {
let row = yield;
if (row) {
user.lid = row[0].value;
debug('Users Log Entry = %d',user.lid);
}
});
}
reply(true,user);
}
})
.catch((err) => {
logger('database','Error on logon: ' + err.message);
reply(false,false);
});
});
There is a quite simple way to do async function in Commander.js
async function run() {
/* code goes here */
}
program
.command('gettime')
.action(run);
program.parse(process.argv);

How can i write a mocha test for the following function?

I want to write a test for this node.js funcion,This has two arguments request and response. I set the request variable . But dont know how to set response variable.
function addCustomerData(request, response) {
common.getCustomerByMobile(request.param('mobile_phone'), function (customerdata) {
if (!customerdata) {
var areaInterest = request.param('area_interest');
var customerInfo = {userType: request.param('userType'),
firstName : request.param('first_name'),
middleName : request.param('middle_name'),
lastName : request.param('last_name'),
email : request.param('email'),
mobilePhone : request.param('mobile_phone'),
uniqueName : request.param('user_name'),
company : request.param('company')
};
if(customerInfo.email){
customerInfo.email = customerInfo.email.toLowerCase();
}
if(customerInfo.uniqueName){
customerInfo.uniqueName = customerInfo.uniqueName.toLowerCase();
}
if(areaInterest) {
customerInfo.areaInterest = '{' + areaInterest + '}';
}else
areaInterest = null;
addCustomer(request, response, customerInfo, function (data) {
request.session.userId = data;
return response.send({success: true, message: 'Inserted successfully'});
}
);
} else {
return response.send({success: false, message: 'User with this mobile number already exists'});
}
});
}
I wrote the test as follows
describe('signup', function(){
describe('#addCustomer()', function(){
before(function (done) {
request = {};
request.data = {};
request.session = {};
request.data['userType'] = '3';
request.data['first_name'] = 'Shiji';
request.data['middle_name'] = '';
request.data['last_name'] = 'George';
request.data['email'] = 'shiji#lastplot.com';
request.data['mobile_phone'] = '5544332333';
request.data['user_name'] = 'shiji';
request.session['imageArray'] = [];
request.param=function(key){
// Look up key in data
return this.data[key];
};
request1 = {};
request1.data = {};
request1.session = {};
request1.data['area_interest']=["aluva","ernakulam"];
request1.data['userType'] = '1';
request1.data['first_name'] = 'Hari';
request1.data['middle_name'] = 'G';
request1.data['last_name'] = 'Ganesh';
request1.data['email'] = 'hari#lastplot.com';
request1.data['mobile_phone'] = '5544332321';
request1.data['user_name'] = 'hariganesh';
request1.data['company'] = 'HG Realestate';
request1.session['imageArray'] = [];
request1.param=function(key){
// Look up key in data
return this.data[key];
};
done();
});
it('It should list the matching properties', function(done){
async.parallel([
function(callback) {
signup.addCustomerData(request, response, function (result, err) {
should.not.exist(err);
should.exists(result);
callback();
});
},
function(callback) {
signup.addCustomerData(request1, response, function (result, err) {
should.not.exist(err);
should.exists(result);
callback();
});
}],function(){
done();
});
});
But i got the error as response has no method send()
Thanks in Advance.
Your addCustomerData function does not have a callback, it just calls respond.send(). You need to mock the response object, as well as the send method, and put your tests inside of it, but you won't be able to use async.parallel() as, like I already mentioned, your function does not have a callback parameter. If you're testing request/response functions, I suggest you look into Supertest https://github.com/visionmedia/supertest which is widely used for cases like this.

mongodb data async in a for-loop with callbacks?

Here is a sample of the working async code without mongodb. The problem is, if i replace the vars (data1_nodb,...) with the db.collection.find(); function, all needed db vars received at the end and the for()-loop ends not correct. Hope someone can help. OA
var calc = new Array();
function mach1(callback){
error_buy = 0;
// some vars
for(var x_c99 = 0; x_c99 < array_temp_check0.length;x_c99++){
// some vars
calc[x_c99] = new Array();
calc[x_c99][0]= new Array();
calc[x_c99][0][0] = "dummy1";
calc[x_c99][0][1] = "dummy2";
calc[x_c99][0][2] = "dummy3";
calc[x_c99][0][3] = "dummy4";
calc[x_c99][0][4] = "dummy5";
function start_query(callback) {
data1_nodb = "data1";
data2_nodb = "data2";
data3_nodb = "data3";
data4_nodb = "data4";
calc[x_c99][0][0] = data1_nodb;
calc[x_c99][0][1] = data2_nodb;
calc[x_c99][0][2] = data3_nodb;
callback(data1_nodb,data2_nodb,etc..);
}
start_query(function() {
console.log("start_query OK!");
function start_query2(callback) {
data4_nodb = "data5";
data5_nodb = "data6";
data6_nodb = "data7";
calc[x_c99][0][3] = data4_nodb;
calc[x_c99][0][4] = data5_nodb;
callback(data5_nodb,data6_nodb,etc..);
}
start_query2(function() {
console.log("start_query2 OK!");
function start_query3(callback) {
for(...){
// do something
}
callback(vars...);
}
start_query3(function() {
console.log("start_query3 OK!");
});
});
});
}
callback(calc);
};
function mach2(callback){
mach1(function() {
console.log("mach1 OK!");
for(...){
// do something
}
});
callback(calc,error_buy);
};
mach2(function() {
console.log("mach2 OK 2!");
});
You need to work with the async nature of the collection.find() method and wait for all of them to be done. A very popular approach is to use the async module. This module allows you run several parallel tasks and wait for them to finish with its async.parallel() method:
async.parallel([
function (callback) {
db.foo.find({}, callback);
},
function (callback) {
db.bar.find({}, callback);
},
function (callback) {
db.baz.find({}, callback);
}
], function (err, results) {
// results[0] is the result of the first query, etc
});

Resources