async function in node only runs after response sent - node.js

I have the method:
const getTaskOrCategoryRecursive = async (type, id)=>{
const rslt = await type.findOne.call(type, {_id:id},{}, standardOptions, err=>{
if(err){
return({err: err});
}
});
const result = rslt.toObject();
const tasks = result.tasks;
const events = result.events;
result.children = {};
result.children.tasks = tasks.map(async taskId=>{
const taskIdString = taskId.toString();
const rtrn = await getTaskRecursive(taskIdString, err=>{
if(err){
return({err: err});
}
});
return rtrn;
});
result.children.events = events.map(async eventId=>{
const eventIdString = eventId.toString();
await Event.findOne({_id:eventIdString}, standardOptions,err=>{
if(err){
return({err: err});
}
});
});
return result;
}
It's called by two methods:
const getTaskRecursive = (taskId)=>{
return getTaskOrCategoryRecursive(Task, taskId)
}
and
const getCategoryRecursive=(categoryId)=>{
return getTaskOrCategoryRecursive(Category,categoryId);
};
which are called by the function
router.get('/:id', verifyToken, async (req,res)=>{
Branch.getCategoryRecursive(req.params.id)
.then(
(cat)=>{
res.status(200).send(cat);
},
(err)=>{
res.status(500).send(err);
}
)
});
When I try to run the program, first the getCategoryRecursive method runs, then res.status(200).send(cat); then the getTasksRecursive method which gets the children of the object being sent in the response. getTasksRecursive does what it is supposed to, but it's running after the response is sent so the object is empty.
How do I make my asynchronous method run before the response is sent?
UPDATE: Using Aritra Chakraborty's answer, I changed it to the following and it worked.
First I separated the .map into a new function:
const getAllTasksRecursive = async(taskIds)=>{
const rtrn = await Promise.all(
taskIds.map(
async taskId=>{
return await getTaskRecursive(taskId.toString(), err=>{if(err){return {err:err}}});
}
)
)
return rtrn;
}
Then I called it in the previous function using:
result.children.tasks = await getAllTasksRecursive(tasks);
Now I am getting the data back in the response.

That's because internally a map or foreach can look something like this:
Array.prototype.forEach = function (callback) {
// this represents our array
for (let index = 0; index < this.length; index++) {
// We call the callback for each entry
callback(this[index], index, this)
}
}
It will call the callback alright, but it will not wait for the callback to complete before running the next one.
So, you need one async foreach of some sort,
async function asyncForEach(array, callback) {
for (let index = 0; index < array.length; index++) {
await callback(array[index], index, array)
}
}
Note, how the callback is being awaited.
Your code will have to be something like this:
const start = async () => {
await asyncForEach([1, 2, 3], async (num) => {
await waitFor(50)
console.log(num)
})
console.log('Done')
}
start()
Refer: https://codeburst.io/javascript-async-await-with-foreach-b6ba62bbf404 This article helped me alot in the past.

Related

NodeJS await all url-exists before returning

I'm having an issue of my controller returning data before url-exists finishes running.
const urlExists = require('url-exists');
const ESAPI = require('node-esapi');
exports.getDocs = async (req, res, next) => {
try {
const id = req.params.tID;
let docs = [];
const getDocs = await models.Documents.getDocs(id); // Getting data from Database
for(const d of getDocs) {
const docName = ESAPI.encoder().encodeForHTML(d.DocumentName);
const path = `https://mywebsite/Files/${id}/${docName}`;
urlExists(path, function(err, exists) {
console.log(exists);
if(exists) {
docs.push({
path,
audited: d.Audited,
comment: d.Comment
});
}
});
}
console.log(docs);
return res.json(docs);
} catch(err) {
return res.json([]);
}
}
I can see in the console.logs() that it first logs the console.log(docs); an empty array. Then, it logs the console.log(exists). How can I wait until the for loop and urlExists finishes running before returning the docs array?
Thanks
urlExists is a callback-based function, you can promisify it and then await it.
To promisify urlExists function, you can use built-in node module: util.promisify.
// import the "util" module
const util = require('util');
// create a promise wrapper around "urlExists" function
const promisifiedUrlExists = util.promisify(urlExists);
After urlExists has been promisified, await it inside the loop.
exports.getDocs = async (req, res, next) => {
try {
...
for(const d of getDocs) {
...
const exists = await promisifiedUrlExists(path);
if(exists) {
docs.push({
path,
audited: d.Audited,
comment: d.Comment
});
}
}
return res.json(docs);
} catch(err) {
return res.json([]);
}
}

Promise.all returning undefined in Node JS

I have a code to fetch directory names from first API. For every directory, need to get the file name from a second API. I am using something like this in my Node JS code -
async function main_function(req, res) {
const response = await fetch(...)
.then((response) => {
if (response.ok) {
return response.text();
} else {
return "";
}
})
.then((data) => {
dirs = ...some logic to extract number of directories...
const tempPromises = [];
for (i = 0; i < dirs.length; i++) {
tempPromises.push(getFilename(i));
}
console.log(tempPromises); // Prints [ Promise { <pending> } ]
Promise.all(tempPromises).then((result_new) => {
console.log(result_new); // This prints "undefined"
res.send({ status: "ok" });
});
});
}
async function getFilename(inp_a) {
const response = await fetch(...)
.then((response) => {
if (response.ok) {
return response.text();
} else {
return "";
}
})
.then((data) => {
return new Promise((resolve) => {
resolve("Temp Name");
});
});
}
What am I missing here?
Your getFilename() doesn't seem to be returning anything i.e it's returning undefined. Try returning response at the end of the function,
async function getFilename(inp_a) {
const response = ...
return response;
}
Thanks to Mat J for the comment. I was able to simplify my code and also learn when no to use chaining.
Also thanks to Shadab's answer which helped me know that async function always returns a promise and it was that default promise being returned and not the actual string. Wasn't aware of that. (I am pretty new to JS)
Here's my final code/logic which works -
async function main_function(req,res){
try{
const response = await fetch(...)
const resp = await response.text();
dirs = ...some logic to extract number of directories...
const tempPromises = [];
for (i = 0; i < dirs.length; i++) {
tempPromises.push(getFilename(i));
}
Promise.all(tempPromises).then((result_new) => {
console.log(result_new);
res.send({ status: "ok" });
});
}
catch(err){
console.log(err)
res.send({"status" : "error"})
}
}
async function getFilename(inp_a) {
const response = await fetch(...)
respText = await response.text();
return("Temp Name"); //
}

Trying to use async.parallelLimit to make to make successive parallel API calls

I have a data array of objects like [{number:1}, {number:2}, {number:3}... {number:100}]. And want to make parallel API calls in successive batches of 10 until the whole array has been processed.
How would I go about that?
Here's my code. It goes over the first 10, but then it stops.
const async = require("async");
const axios = require("axios");
const calls = require("../model/data"); // [{number:1}, {number:2},{number:3},...{number:100}]
let makeAPICall = function (request, callback) {
axios
.post("http://www.api.com/", {
number: `${request.number}`,
webhookURL: "http://localhost:8000/",
})
.then(function (response) {})
.catch(function (err) {
callback(err);
});
};
const functionArray = calls.map((request) => {
return (callback) => makeAPICall(request, callback);
});
exports.startCalls = (req, res, next) => {
async.parallelLimit(functionArray, 10, (err, results) => {
if (err) {
console.error("Error: ", err);
} else {
console.log("Results: ", results.length, results);
}
});
};
So, your approach is somewhat backwards as you have something that already returns a promise (Axios) which is the modern way to manage asynchronous operations and now you're trying to convert it back to a plain callback so you can use an old-fashioned library (the async library). It's also slower to run 10, wait for all 10 to finish before starting any more when that is typically not necessary.
Instead, I would suggest you use the Axios promise you already have. You can either use Bluebird's .map() which has a concurrency setting or you can use this bit of code that gives you a function for running promise-producing functions in parallel while controlling the max number that are in-flight at any given time:
// takes an array of items and a function that returns a promise
function mapConcurrent(items, maxConcurrent, fn) {
let index = 0;
let inFlightCntr = 0;
let doneCntr = 0;
let results = new Array(items.length);
let stop = false;
return new Promise(function(resolve, reject) {
function runNext() {
let i = index;
++inFlightCntr;
fn(items[index], index++).then(function(val) {
++doneCntr;
--inFlightCntr;
results[i] = val;
run();
}, function(err) {
// set flag so we don't launch any more requests
stop = true;
reject(err);
});
}
function run() {
// launch as many as we're allowed to
while (!stop && inflightCntr < maxConcurrent && index < items.length) {
runNext();
}
// if all are done, then resolve parent promise with results
if (doneCntr === items.length) {
resolve(results);
}
}
run();
});
}
You would then use it like this:
let makeAPICall = function(request) {
return axios.post("http://www.api.com/", {
number: `${request.number}`,
webhookURL: "http://localhost:8000/",
});
};
mapConcurrent(calls, 10, makeAPICall).then(results => {
// all results in order here
console.log(results);
}).catch(err => {
console.log(err);
});
See another similar issue here: Promise.all consumes all my RAM
If you really want to run them in fixed batches where the whole batch finishes before you run any more requests, you could do something like this:
const axios = require("axios");
const calls = require("../model/data"); // [{number:1}, {number:2},{number:3},...{number:100}]
function makeAPICall(request) {
return axios.post("http://www.api.com/", {
number: `${request.number}`,
webhookURL: "http://localhost:8000/",
});
};
async function runBatches(array, batchSize, fn) {
let index = 0;
let results = [];
while (index < array.length) {
let promises = [];
for (let num = 0; num < batchSize && index < array.length; ++num) {
promises.push(makeAPICall(array[index++]));
}
let batchResults = await Promise.all(promises);
results.push(...batchResults);
}
return results;
}
runBatches(calls, 10, makeAPICall).then(results => {
// all results in order here
console.log(results);
}).catch(err => {
console.log(err);
});

Get value out of function in Node.js

I don't know about the correct title for my problem. I just need a value going out of a function, just like return but I think it's not same.
i have code snippet from controller in adonisjs framework:
var nmeadata = "";
jsondata.forEach(element => {
var x = element.nmea
var buff = new Buffer(x, 'base64')
zlib.unzip(buff, (err, res) => {
if(err)
{
//
}
else
{
nmeadata += "greed island"
nmeadata += res.toString()
}
})
});
return view.render('admin.index', {
data: datanmea.toJSON(),
nmea: nmeadata
})
I need the result of unzipped string data that inserted to nmeadata from zlib function then send it to view. But, for this time, even I cannot displaying a simple output like greed island to my view.
thank you.
UPDATE
Still not working after using promises:
class NmeaController {
async index({view})
{
const datanmea = await NmeaModel.all()
const jsondata = datanmea.toJSON()
var promises = [];
var nmeadata = "";
jsondata.forEach(element => {
promises.push(
new Promise(resolve => {
let x = element.nmea
let buff = new Buffer(x, 'base64')
zlib.unzip(buff,
(err, res) => {
if (err) {
//
} else {
nmeadata += "test add text"
// nmeadata += res.toString()
}
//im also try using resolve() and resolve("any text")
resolve(nmeadata);
})
}
)
)
});
await Promise.all(promises);
return view.render('admin.index', {
data: datanmea.toJSON(),
nmea: nmeadata
});
}
UPDATE AUGUST 22 2019
i'm already tried solution from maksbd19 but still not working
class NmeaController {
async index({view})
{
const datanmea = await NmeaModel.all()
const jsondata = datanmea.toJSON()
var promises = [];
var nmeadata = "";
jsondata.forEach(element => {
promises.push(
new Promise(resolve => {
let x = element.nmea
let buff = new Buffer(x, 'base64')
zlib.unzip(buff,
(err, res) => {
if (err) {
// since you are interested in the text only, so no need to reject here
return resolve("");
}
return resolve("greed island")
})
}
)
)
});
const result = await Promise.all(promises); // this will be an array of results of each promises respectively.
nmeadata = result.join(""); // process the array of result
return view.render('admin.index', {
data: datanmea.toJSON(),
nmea: nmeadata
});
}
}
I'd suggest two things-
modify zlib.unzip callback function to resolve properly;
(err, res) => {
if (err) {
// since you are interested in the text only, so no need to reject here
return resolve("");
}
return resolve(res.toString())
}
retrieve the final data from the result of Promise.all
const result = await Promise.all(promises); // this will be an array of results of each promises respectively.
nmeadata = result.join(""); // process the array of result
In this approach every promise will resolve and finally you will get the expected result in array.

Return value after all the promises are resolved

I am working on a nodejs code that fetches data from a site, parses it, finds particular data and fetches something else for the data that was previously fetched. But the final return statement is returning without the value fetched from the second API call.
I tried to implement async await, but I am not sure where do I have to put them exactly.
const getMainData = async val => {
let result = [];
//get xml data from the API
const xmlData = await getSiteContent(`value`); //axios call
parseString(xmlData, (err, json) => { //convert xml to json
const { entry } = json.feed; // array of results.
result = entry.map(report => {
const secondInfo = getSomeMoreData(report.something); //axios call
const data = {
id: report.id,
date: report.date,
title: report.title
};
data.info = secondInfo;
return data;
});
});
return { result };
};
I was expecting the function to return the array result that has id, date, title and info. But I am getting info as null since it is going to another function that does one more API call.
Try wrapping parseString in a promise so you can await the result, then make the entry.map callback an async function so that you can use the await keyword to wait for the result of the axios fetch.
async function xml2json(xml) {
return new Promise((resolve, reject) => {
parseString(xml, function (err, json) {
if (err)
reject(err);
else
resolve(json);
});
});
}
const getMainData = async val => {
//get xml data from the API
const xmlData = await getSiteContent(`value`); //axios call
const json = await xml2json(xmlData);
const { entry } = json.feed; // array of results
const result = await Promise.all(
entry.map(async report => {
const secondInfo = await getSomeMoreData(report.something); // axios call
const data = {
id: report.id,
date: report.date,
title: report.title,
};
data.info = secondInfo;
return data;
})
)
return { result };
}
Let me know if that works. If not, I can try to help you out further.
The problem with your code is you have mixed promises concept(async/await is a syntactic sugar - so same thing) along with callback concept.
And the return statement is outside callback() of parseString() and the callback would be executed maybe after returning results only because parseString() is an asynchronous function.
So in the following solution I have wrapped parseString() in a promise so that it can be awaited.
const parseStringPromisified = async xmlData => {
return new Promise((resolve, reject) => {
parseString(xmlData, (err, json) => {
if (err) {
reject(err);
}
resolve(json);
});
});
};
const getMainData = async val => {
//get xml data from the API
const xmlData = await getSiteContent(`value`); //axios call
const json = await parseStringPromisified(xmlData);
const { entry } = json.feed; // array of results.
const result = entry.map(async report => {
const secondInfo = await getSomeMoreData(report.something); //axios call
return {
id: report.id,
date: report.date,
title: report.title,
info: secondInfo
};
});
return Promises.all(result);
};

Resources