async.parallel doesn't wait for the parallel functions - node.js

Running the code bellow I just get "undefined". It seems like the async.parallel call executes the final function before the 4 parallel functions are finished.
function classify(filename) {
var stack = [];
var functionOne = async function(callback){
//call to an API
res = await classifierOne(filename);
callback(null, res);
}
var functionTwo = async function(callback){
//call to an API
res = await classifierTwo(filename);
callback(null, res);
}
var functionThree = async function(callback){
//call to an API
res = await classifierThree(filename);
callback(null, res);
}
var functionFour = async function(callback){
//call to an API
res = await classifierFour(filename);
callback(null, res);
}
stack.push(functionOne);
stack.push(functionTwo);
stack.push(functionThree);
stack.push(functionFour);
async.parallel(stack, function(err, res){
console.log(res[0]);
})
}

If you are using async with version 3.x, we have a big change - Support async/await , with the version, with tasks which have been wrap to a Promise (when you use async/await keyword) then the callback will be disable, you need to return value in the tasks instead of call callback(null, value).
This mean, to fix your case, the code will come to:
function classify(filename) {
var stack = [];
var functionOne = async function(){ // remove callback param
//call to an API
res = await classifierOne(filename);
return res; // return value
}
var functionTwo = async function(){
//call to an API
res = await classifierTwo(filename);
return res;
}
var functionThree = async function(){
//call to an API
res = await classifierThree(filename);
return res;
}
var functionFour = async function(){
//call to an API
res = await classifierFour(filename);
return res;
}
stack.push(functionOne);
stack.push(functionTwo);
stack.push(functionThree);
stack.push(functionFour);
async.parallel(stack, function(err, res){
console.log(res[0]);
})
}
P/s: My suggestion is using Promise.all for this case. Say no to callback.
async function classify(filename) { // Make this function become a async function
var stack = [];
var functionOne = async function(){ // remove callback param
//call to an API
res = await classifierOne(filename);
return res; // return value
}
var functionTwo = async function(){
//call to an API
res = await classifierTwo(filename);
return res;
}
var functionThree = async function(){
//call to an API
res = await classifierThree(filename);
return res;
}
var functionFour = async function(){
//call to an API
res = await classifierFour(filename);
return res;
}
stack.push(functionOne);
stack.push(functionTwo);
stack.push(functionThree);
stack.push(functionFour);
const result = await Promise.all(stack); // Wait until all "task" finish
console.log(result);
return result; // Return the result of all tasks
}
// Call classify function inner a async function
const result = await classify('your_file_name'); // await keyword
console.log(result);

I am not too sure about the async npm library but on top of it looks like it doesn't work when you have a callback function as async. If you try it using native promises, it will work. Something like
function classify(filename) {
var stack = [];
var functionOne = function(callback) {
//call to an API
classifierOne(filename).then(res => callback(null, res));
};
var functionTwo = function(callback) {
classifierTwo(filename).then(res => callback(null, res));
};
var functionThree = function(callback) {
classifierThree(filename).then(res => callback(null, res));
};
var functionFour = function(callback) {
classifierFour(filename).then(res => callback(null, res));
};
stack.push(functionOne);
stack.push(functionTwo);
stack.push(functionThree);
stack.push(functionFour);
async.parallel(stack, function(err, res) {
console.log(res);
});
}

Your methods like i.e. functionOne missed the return and didn't a callback. The below returns ["1","2","3","4"].
const async = require('async');
const classifierOne = async ()=> "1"
const classifierTwo = async ()=> "2"
const classifierThree = async ()=> "3"
const classifierFour = async ()=> "4"
const classify = filename => {
var stack = [];
var functionOne = async ()=> classifierOne(filename);
var functionTwo = async ()=> classifierTwo(filename);
var functionThree = async ()=> classifierThree(filename);
var functionFour = async ()=> classifierFour(filename);
stack.push(functionOne);
stack.push(functionTwo);
stack.push(functionThree);
stack.push(functionFour);
async.parallel(stack, (err, res)=>{
console.log(res);
})
}
classify("x");

Related

How to unit test nested function in node js?

I have a function for which I am writing unit test but that function is calling another function and there I am not able to mock/stub that function.
for example :
function getValue( param1, param2, callback){
getData(param1, param3).then( response) => {
return callback();
}, (err) => {
return callback();
});
}
So I don't know how to mock getData() function.
Here is a working example that demonstrates what you are trying to do:
lib.js
function getData(param1, param2) {
return fetch('someUrl'); // <= something that returns a Promise
}
exports.getData = getData;
code.js
const lib = require('./lib');
export function getValue(param1, param2, callback) {
return lib.getData(param1, param2).then(response => {
callback(response);
}).catch(err => {
callback(err);
});
}
exports.getValue = getValue;
code.test.js
const sinon = require('sinon');
const lib = require('./lib');
const { getValue } = require('./code');
describe('getValue', () => {
it('should do something', async () => {
const stub = sinon.stub(lib, 'getData');
stub.resolves('mocked response');
const callback = sinon.spy();
await getValue('val1', 'val2', callback);
sinon.assert.calledWithExactly(stub, 'val1', 'val2'); // Success!
sinon.assert.calledWithExactly(callback, 'mocked response'); // Success!
});
});
Update
OP added in the comments that they can't use async / await and are exporting the function using module.exports = getData;.
In that case the module export is the function and the entire module needs to be mocked with something like proxyquire.
The assertions should be done in a then callback and the test should return the resulting Promise so mocha knows to wait for it to resolve.
Updated example:
lib.js
function getData(param1, param2) {
return fetch('someUrl'); // <= something that returns a Promise
}
module.exports = getData;
code.js
const getData = require('./lib');
function getValue(param1, param2, callback) {
return getData(param1, param2).then(response => {
callback(response);
}).catch(err => {
callback(err);
});
}
module.exports = getValue;
code.test.js
const sinon = require('sinon');
const proxyquire = require('proxyquire');
describe('getValue', () => {
it('should do something', () => {
const stub = sinon.stub();
stub.resolves('mocked response');
const getValue = proxyquire('./code', { './lib': stub });
const callback = sinon.spy();
return getValue('val1', 'val2', callback).then(() => {
sinon.assert.calledWithExactly(stub, 'val1', 'val2'); // Success!
sinon.assert.calledWithExactly(callback, 'mocked response'); // Success!
});
});
});
function getValue( param1, param2, callback){
getData(param1, param3).then( response) => {
callback(response);
});
}
getvalue(param1, param2, function(error, response)) {
console.log(response)
}
It might help you.

NodeJs async.waterfall task.apply is not a function

I am trying to learn NodeJs's async library,
I created 3 functions and pushed them into an array,
the array is then passed to the async.waterfall function,
But I get the following error -
Following is my code -
var async = require('async');
var waterfallFunctionArray = [];
var functionOne = function(callback) {
console.log("WATERFALL ONE");
callback(null, 1);
};
var functionTwo = function(param1, callback) {
console.log("WATERFALL TWO");
callback(null, param1+param1);
};
var functionThree = function(param2, callback) {
console.log("WATERFALL THREE");
callback(null, param2+1);
};
waterfallFunctionArray.push(functionOne);
waterfallFunctionArray.push(functionTwo);
waterfallFunctionArray.push(functionThree);
async.waterfall([waterfallFunctionArray], function (err, result) {
if (err) {
console.error(err);
return;
}
console.log("WATERFALL RESULT => \n"+result);
});
[waterfallFunctionArray] == [[]]

Function enters then before request completed

My async function enters then before request is completed. Shouldn't Then part of the code executes only after the async function is completed? How to make the function call only when all the function has finished executing?
app.js
var request_test = require('./request_test');
baseUrl = "https://github.com";
promiseFunction().then((result)=>{
console.log("complete")
});
var promiseFunction = async function promiseFunction() {
request_test.request_test(baseUrl);
}
request_test.js
var request = require('request');
var cheerio = require('cheerio');
var request_test = function check(baseUrl) {
console.log("baseUrl:" + baseUrl)
var options = {
url: baseUrl
};
request(options, function (error, response, html) {
if (!error) {
console.log("no error");
}else{
console.log("else js");
console.log(error);
}
});
}
module.exports = {
request_test: request_test
};
In order to use then() you need to return a promise. So here is an example of the good old style promise chain, simply return promise from request_test and once you resolve or reject it, then() will be called:
promiseFunction().then((result) => {
console.log("complete");
});
function promiseFunction() {
return request_test();
}
function request_test() {
return new Promise(function(resolve, reject) {
setTimeout(function() {
console.log("no error");
resolve();
}, 1000);
});
}
Or maybe use the modern approach - async method to await call function that returns promise.
promiseFunction();
async function promiseFunction() {
await request_test();
console.log('complete');
}
function request_test() {
return new Promise(function(resolve, reject) {
setTimeout(function() {
console.log("no error");
resolve();
}, 1000);
});
}
Your issue is coming from var request_test = function check(baseUrl) { ... inside this function you are not returning promise, you are even returning nothing :)
if you are using async I would go ahead and use the await/async syntax. Also the package request does not return a promise, you have an alternative with request-promise-native. The promise should be the return value of your helper function. It could look like this:
var request_test = require('./request_test');
var baseUrl = "https://github.com";
var promiseFunction = async function () {
var result = await request_test.request_test(baseUrl);
console.log("complete");
}
promiseFunction();
and the module:
var request = require('request-promise-native');
var cheerio = require('cheerio');
var request_test = function check(baseUrl) {
console.log("baseUrl:" + baseUrl)
var options = {
url: baseUrl
};
return request(options).then(function (error, response, html) {
if (!error) {
console.log("no error");
} else{
console.log("else js");
console.log(error);
}
});
}
module.exports = {
request_test: request_test
};

Async parallel and promises wont work

I have the following two snippets:
1.this is the unit I would like to run:
health.checkHealth = function (req, res) {
async.parallel({
cassandra: function (callback) {
cassandraRepository.ping(callback);
},
kafka: function (callback) {
kafkaHelper.ping(callback);
},
vault: function (callback) {
vaultClient.health()
.then((isOk)=>{
callback(null, isOk)})
.catch(()=>
callback(null, false));
}
}, function (err, result) {
var response = {};
if (result.cassandra) {
response.cassandra = "OK";
} else {
response.cassandra = "Failed";
res.status(500);
}
if (result.kafka) {
response.kafka = "OK";
} else {
response.kafka = "Failed";
res.status(500);
}
if (result.vault) {
response.vault = "OK";
} else {
response.vault = "Failed";
res.status(500);
}
res.send(response);
})
}
2.this is the test to check the unit:
describe('vault is avaliable', function () {
beforeEach(sinon.test(function () {
sinon.stub(vaultClient, "health").resolves(true);
}));
it('should return 200', sinon.test(function (done) {
var response = {
vault: "OK"
};
var req = {};
var res = {};
var spySend = res.send = sinon.spy();
var spyStatus = res.status = sinon.spy();
health.checkHealth(req, res);
expect(spySend.calledOnce).to.equal(true);
expect(spySend.calledWith(response));
expect(spyStatus.calledOnce).to.equal(false);
}));
});
My problem is that when I call checkHealth it proceeds to the next line (expect(spySend.calledOnce).to.equal(true);) without waiting for the vaultClient's promise to complete.
What do I need to do to make the expects run only after the 'checkHealth' was run.
You do not need to use the async - you can use promises directly in your code with Promise.all as others have suggested.
The code here is using promisifyAll from Bluebird, but you can also convert the APIs to use promises yourself.
//.props is bluebird, you can `.all` with native promises too
Promise.props({
cassandra: cassandraRepository.pingAsync(); // created by promisifyAll
kafka: kafkaHelper.pingAsync(),
vault: vaultClient.health()
}).then(results => {
// access results.cassandra, results.kafka and results.vaule here
});

Chaining multiple chained promises with Q (loopback app)

Here is a scenario, I've implemented a loopback remote method which imports some data from REST connector to local postgresql connector.
I can do this for a single model
var importData = function (model, cb) {
migrateModel(model, cb)
.then(findImportInfo)
.then(fetchRemoteData)
.then(processFetchedData)
.then(updateImportInfo)
.then(countLocalData)
.then(importCompleted)
.catch(function (err) {
importFailed(err, cb);
})
.done(function () {
console.log('done');
});
};
So the chain does many thing and at the end importCompleted calls the provide cb which is the callback that returns the response to the REST API.
But I can't figure how to do this with multiple models and return each result. I tried something like this, it works actually but REST API never receives a result.
var importDataAll = function (app, cb) {
var models = app.models();
var deferred = Q.defer();
var promises = [];
var results = [];
function doCallback() {
cb(null, results);
}
models.forEach(function (model) {
if (typeof model.importData === 'function') {
migrateModel(model, model.definition.name, null)
.then(findImportInfo)
.then(fetchRemoteData)
.then(processFetchedData)
.then(updateImportInfo)
.then(countLocalData)
.then(function (prevResult) {
var deferred = Q.defer();
var remoteCount = prevResult.dataCount;
var localCount = prevResult.recordCount;
var result =
{
'time': new Date(),
'remoteCount': remoteCount,
'localCount': localCount,
'started': prevResult.started,
'completed': new Date()
}
results.push(result);
deferred.resolve(result);
return deferred.promise;
})
.catch(function (err) {
promises.reject(err);
})
}
});
return Q.allSettled(promises).then(doCallback);
};
I'm lost at that point, any ideas?
EDIT
Trying #Otze's answer I tried this also
var importDataAll = function (app, cb) {
var models = app.models().filter(function (element, index, array) {
return typeof element.importData === 'function';
});
var promises = models.map(function (model) {
migrateModel(model, model.definition.name, null)
.then(findImportInfo)
.then(fetchRemoteData)
.then(processFetchedData)
.then(updateImportInfo)
.then(countLocalData)
.then(importResult)
.catch(function (err) {
promises.reject(err);
})
});
Q.all(promises)
.then(function (resolvedPromises) {
cb(null, results);
});
};
But the result is the same, cb gets called early but the code actually runs in order. I just can't get the result to the response. I think it's never ends so the REST API gets no content after some time.
Have a look at Q.all or any of the other promise combination functions:
http://documentup.com/kriskowal/q/#combination
With Q.all you could do somehting like this:
var promises = myModels.map(doAllThePromiseThings);
Q.all(promises)
.then(function(resolvedPromises) {
doStuff();
});
Note that you need to return a promise from doAllThePromiseThings.
Since .then returns a promise you can simply do:
.then(function (prevResult) {
return {
'time': new Date(),
'remoteCount': prevResult.dataCount,
'localCount': prevResult.recordCount,
'started': prevResult.started,
'completed': new Date()
};
})
instead of
.then(function (prevResult) {
var deferred = Q.defer();
var remoteCount = prevResult.dataCount;
var localCount = prevResult.recordCount;
var result =
{
'time': new Date(),
'remoteCount': remoteCount,
'localCount': localCount,
'started': prevResult.started,
'completed': new Date()
}
results.push(result);
deferred.resolve(result);
return deferred.promise;
})
I use bluebird library's map method to accomplish such use cases:
https://github.com/petkaantonov/bluebird/blob/master/API.md#mapfunction-mapper--object-options---promise
var Promise = require('bluebird');
var importDataAll = function (app, cb) {
var models = app.models().filter(function (element, index, array) {
return typeof element.importData === 'function';
});
Promise.map(
models,
function(model) {
return migrateModel(model, model.definition.name, null) // don't want to break the promise chain
.then(findImportInfo)
.then(fetchRemoteData)
.then(processFetchedData)
.then(updateImportInfo)
.then(countLocalData)
.then(importResult)
.then(function(){
...
return Promise.resolve(); // don't want to break the promise chain
});
},
{concurrency: 1}
)
.then(function () {
debug('finished working on all the models one-by-one');
cb(null);
})
.catch(function (err) {
cb(err);
});

Resources