How to return an object after multiple requests are done - node.js

If I didn't need to worry about asynchronous code, this is what it'll look like:
app.get('/user/:username/events', async (req, res) => {
// Grab all events by that user
let events = await axios.get(`${baseApiUrl}/users/${req.params.username}/events`, options).then(d => d.data)
// Loop through events
events.forEach(event => {
// If event type is A_Event
if (event.type === 'A_Event') {
// Loop through objArr array
event.payload.objArr.forEach(async obj => {
// Grab additional data through endpoint
// that lives in obj
let data = await axios.get(obj.url)
// Append data back onto obj
obj.objData = data
})
}
})
// return events obj with objData appended to each obj within
// objArr
res.json({ events })
})
But this doesnt work because it returns events before its done grabbing all the data.
I tried doing something with Promise.all but couldnt get what I needed to get.
events.map(async event => {
if (event.type === 'A_Event') {
getData(event.payload.objArr)
.then(objData => {
// This returns an array of objData that
// then needs to be added back to the specific obj
// it came from
event.payload.objArr.forEach((obj, i) => {
obj.objData = objData[i]
})
})
}
res.json({ events })
})
const getData = async objArr => {
const requests = objArr.map(async obj => {
const data = await axios.get(obj.url, options)
return {
stats: data.data.stats,
created_at: data.data.created_at
}
})
return Promise.all(requests)
}
Would really appreciate the help with this. Been stuck on this problem for a good bit.
EDIT
Tried switching to a for of loop according to this post: Using async/await with a forEach loop... No luck but I think its a step in the right direction.
events.forEach(async event => {
if (event.type === 'A_Event') {
for (let obj of event.payload.objArr) {
let objData = await axios.get(obj.url, options)
obj.objData = objData
console.log('setting value')
}
}
})
res.json({ events })
console.log('returned value')
I get returned value before I get setting value...

Promise.all(requests);
Will return a promise so you can handle it somewhere else like this:
getData(objaArr).then((result) => {
// your code
});

In your second block of code, just await for Promise.all()
return await Promise.all(requests);

You would use Promise.all as suggested in the other answer(s).
To retrieve the result you would do something like:
let [languages, currencies, countries] = await Promise.all([
this.downloadLanguages(),
this.downloadCurrencies(),
this.downloadCountries()
])
Your variables can be assigned values returned from the promises using this call structure.

Got it working. I needed to switch the initial loop to a for of. Not just the inner loop. Thanks everyone!
for (let event of events) {
if (event.type === 'A_Event') {
for ( let obj of event.payload.objArr) {
let objData = await axios.get(obj.url, options)
obj.objData = objData.data
}
}
}

You could use good old-fashioned for loops:
for (let i = 0; i < events.length; i++) {
if (events[i].type === 'A_Event') {
for (let j = 0; j < events[i].payload.objArr.length; j++) {
const data = await axios.get(events[i].payload.objArr[j].url);
events[i].payload.objArr[j].objData = data;
}
}
}

Related

NodeJS execute the next process after then() [duplicate]

This question already has answers here:
Using async/await with a forEach loop
(33 answers)
Closed 3 years ago.
I'm trying to do upsert inside forEach becase the request is an array of objects using Sequelize.upsert method. I do this:
async createInfo (req, res){
newsData = req.body.news,
newsList = [];
newsData.forEach(async values => {
var news = {};
news.inserted_id = values.inserted_id;
if(news.inserted_id == null){
do {
news.inserted_id = crypto.getRandom(5);
var check = await InstitutionNews.count({where: {inserted_id: news.inserted_id}});
} while (check > 0);
}
News.upsert({
institution_id: institution_id,
inserted_id: news.inserted_id,
news_date: values.news_date,
news_title: values.news_title,
description: values.news_description,
created_by: created_by
}).then(ResNews => {
news.news_date = values.news_date;
news.news_title = values.news_title;
news.description = values.news_description;
newsList.push(news);
})
})
console.log("TEST")
}
but the process stop at the then(). It didn't execute the next code like the console.log.
Is there any way to execute next line code after the then(). I need then() because I wanna push the news object into newsList array. Because I need newsList as the if else conditional to do the next process.
Thanks.
Since it sounds like you need to wait for forEach to complete before you do another step, so I'd suggest using something like Promise.all:
async function createInfo(req, res) {
const newsData = req.body.news
const newsList = []
try {
await Promise.all(
newsData.map(
async (values) => {
const news = {}
news.inserted_id = values.inserted_id
// ...
News.upsert({...})
.then(ResNews => {
// ...
newsList.push(news)
})
}
)
)
console.log('newsList', newList)
// do other work
} catch (error) {
// handle errors appropriately
}
}
This way, you're creating an array of promises and waiting for all of them to resolve/finish.
Your current approach with forEach won't work in this case since it won't wait for each asynchronous call to finish before executing the "next" step. Since Promise.all returns a single Promise that you can then "wait" for to resolve before continuing with your next step.
Here's a simple example that somewhat does what you're trying to do:
async function createInfo() {
const newsData = [1, 2, 3, 4, 5]
const newsList = []
await Promise.all(
newsData.map(
async (values) => {
const temp = await Promise.resolve('temp')
console.log('first async call inside values of', values)
Promise.resolve('resolved')
.then((result) => {
newsList.push(`resolved with ${values}`)
})
}
)
)
console.log('newsList after')
console.log(newsList)
}
createInfo()
EDIT
Here's an alternative solution as rightly pointed by #Bergi in the comments:
async function createInfo(req, res) {
const newsData = req.body.news
try {
const newList = await Promise.all(
newsData.map(
async (values) => {
const news = {}
news.inserted_id = values.inserted_id
// ...
// since it doesn't look like you're using any
// data that you'd get back from the `News.upsert` call, wait for
// it to finish and just simply return your `news` object
await News.upsert({...})
news.news_date = values.news_date
news.news_title = values.news_title
news.description = values.news_description
return news
}
)
)
console.log('newsList', newList)
// do other work
} catch (error) {
// handle errors appropriately
}
}
use await instead of then like
await News.upsert({
institution_id: institution_id,
inserted_id: news.inserted_id,
news_date: values.news_date,
news_title: values.news_title,
description: values.news_description,
created_by: created_by
});
news.news_date = values.news_date;
news.news_title = values.news_title;
news.description = values.news_description;
newsList.push(news);

Break from async function in await call

Okay, so I'm working with data from Memcache using a promise based library but the issue I'm having is I don't know a way to break from the async call if a result is found?
The code I'm working with is:
const _pong = function() {
return socket.emit('aye', {
pong: globals.uuid()
});
};
return socket.on('helo', async function(data) {
socket._uuid = data.uuid;
let key = 'ws-ping:' + data.uuid;
await cache.get(key).then((result) => {
if(result !== undefined) {
_pong();
}
});
......
});
I basically need to just ignore the rest of the socket.on function if a result is found using the given key? but it seems to continue?
Because you're using await, you can ditch the .then, and get the result directly, in the same block - if the result exists, then just return (after _ponging, if that's the logic you're looking for):
return socket.on('helo', async function(data) {
socket._uuid = data.uuid;
let key = 'ws-ping:' + data.uuid;
const result = await cache.get(key);
if (result !== undefined) {
_pong();
return;
}
// ...
});

Node API - res.send() from 2nd loop

I'm trying to return the hole master/detail object to client, but detail is coming as an empty array
Like this post I've also ended with the same problem:
"Can't call res.send(data) inside the loop because res.send() can only be called once."
"But if I call res.send(array) outside of the loop, the array is still empty"
What is the right way to do it?
I'm trying not to use use asyn
var getMasterDetail = function (req, res) {
const key = "Detail";
var list = {}
list[key] = []
var modelsMaster = objModels.ObjMaster
var modelsDetail = objModels.objDetail
modelsMaster.getMasters(objModels.hdb, (e, master) => {
if (e) {
return console.log(e);
}
for (i = 0; i < master.length; i++) {
modelsDetail.getDetails(objModels.hdb, master[i].nrMaster, (e, detail) => {
if (e) {
return console.log(e);
}
for (j = 0; j < detail.length; j++) {
list[key].push(detail[j])
}
})
master[i].DetailList = list
};
res.send({ MasterDetail: master })
})
};
Thanks.
UPDATE: The answer from #Hammerbot was almost right, but,
I have not notice at the time, that I was getting the same detail for all masters.
Ex. {master:{1,2,3,4,5,6}, master{1,2,3,4,5,6}} instead of {master:{1,2,3}, master{4,5,6}}
I Have no any idea why and how to fix it. I've tried to clean the list befor the loop, and move master master[i].DetailList, creating a second Promisse for the second loop, no success.
You should use promises for that. Here is an example that should resolve your problem:
var getMasterDetail = function (req, res) {
const key = "Detail";
var list = {}
list[key] = []
var modelsMaster = objModels.ObjMaster
var modelsDetail = objModels.objDetail
modelsMaster.getMasters(objModels.hdb, (e, master) => {
if (e) {
return console.log(e);
}
const promises = []
for (i = 0; i < master.length; i++) {
const promise = new Promise(resolve => {
master[i].DetailList = list
modelsDetail.getDetails(objModels.hdb, master[i].nrMaster, (e, detail) => {
if (e) {
return console.log(e);
}
for (j = 0; j < detail.length; j++) {
list[key].push(detail[j])
}
resolve()
})
})
promises.push(promise)
}
Promise.all(promises).then(() => {
res.send({ MasterDetail: master })
})
})
};
As you can see, before the loop I initiate a promises array. Inside the loop, I create a promise by iteration that gets resolved when the callback has finished.
I push the promise into the promises Array, and at the end I use Promise.all() to wait for all the promises to get resolved before sending the result in the response.

insert document into multiple instance of couch DB in Node JS with all success and failure result in any way possible

i have array of db like
const dbArr = ["http://localhost:5984", "http://xyz_couchdb.com:5984"]
data to insert
let data ={
_id: 324567,
name: Harry,
gerder: male
}
here is the logic i am using nano module
return new Promise((resolve, reject) => {
let res = [];
let rej = [];
let counter = 0;
for(let i = 0; i < dbArr.length ; i++){
dbArr[i].insert(data, (err, body) => {
err ? rej.push(err) : res.push(body)
if(counter === obj.dbArray.length -1){
rej.length ? reject(rej) : resolve(res)
}
counter++;
})
}
})
what can be the best possible way to achieve this using promise or async module or anything.
In the following example, we gotta use Array.map to create one promise for each element of dbArr, then we gotta wait all promises to end using Promise.all. The catch is here so we handle the errors.
function getAll(dbArr) {
return Promise.all(dbArr.map(x => x.insert(data)));
}
getAll(dbArr)
.then((rets) => {
// Handle the returns
// They are in an array
})
.catch((err) => {
// Handle the error
});
EDIT :
Ok after checking out the documentation of node-couchdb (the one I suppose you use) - I saw that the .insert() method do not return a Promise but only a callback.
So we gotta transform the method, so it will return a Promise using util.Promisify()
const {
promisify,
} = require('util');
function getAll(dbArr) {
return Promise.all(dbArr.map(x => promisify(x.insert)(data)));
}
getAll(dbArr)
.then((rets) => {
// Handle the returns
// They are in an array
})
.catch((err) => {
// Handle the error
});

Async/Await not waiting

I'm running into an issue which I don't fully understand. I feel like there are likely concepts which I haven't grasped, code that could be optimized, and possibly a bug thrown in for good measure.
To greatly simplify the overall flow:
A request is made to an external API
The returned JSON object is parsed and scanned for link references
If any link references are found, additional requests are made to populate/replace link references with real JSON data
Once all link references have been replaced, the original request is returned and used to build content
Here, is the original request (#1):
await Store.get(Constants.Contentful.ENTRY, Contentful[page.file])
Store.get is represented by:
async get(type, id) {
return await this._get(type, id);
}
Which calls:
_get(type, id) {
return new Promise(async (resolve, reject) => {
var data = _json[id] = _json[id] || await this._api(type, id);
console.log(data)
if(isAsset(data)) {
resolve(data);
} else if(isEntry(data)) {
await this._scan(data);
resolve(data);
} else {
const error = 'Response is not entry/asset.';
console.log(error);
reject(error);
}
});
}
The API call is:
_api(type, id) {
return new Promise((resolve, reject) => {
Request('http://cdn.contentful.com/spaces/' + Constants.Contentful.SPACE + '/' + (!type || type === Constants.Contentful.ENTRY ? 'entries' : 'assets') + '/' + id + '?access_token=' + Constants.Contentful.PRODUCTION_TOKEN, (error, response, data) => {
if(error) {
console.log(error);
reject(error);
} else {
data = JSON.parse(data);
if(data.sys.type === Constants.Contentful.ERROR) {
console.log(data);
reject(data);
} else {
resolve(data);
}
}
});
});
}
When an entry is returned, it is scanned:
_scan(data) {
return new Promise((resolve, reject) => {
if(data && data.fields) {
const keys = Object.keys(data.fields);
keys.forEach(async (key, i) => {
var val = data.fields[key];
if(isLink(val)) {
var child = await this._get(val.sys.linkType.toUpperCase(), val.sys.id);
this._inject(data.fields, key, undefined, child);
} else if(isLinkArray(val)) {
var children = await* val.map(async (link) => await this._get(link.sys.linkType.toUpperCase(), link.sys.id));
children.forEach((child, index) => {
this._inject(data.fields, key, index, child);
});
} else {
await new Promise((resolve) => setTimeout(resolve, 0));
}
if(i === keys.length - 1) {
resolve();
}
});
} else {
const error = 'Required data is unavailable.';
console.log(error);
reject(error);
}
});
}
If link references are found, additional requests are made and then the resulting JSON is injected into the original JSON in place of the reference:
_inject(fields, key, index, data) {
if(isNaN(index)) {
fields[key] = data;
} else {
fields[key][index] = data;
}
}
Notice, I'm using async, await, and Promise's I believe in their intended manor. What ends up happening: The calls for referenced data (gets resulting of _scan) end up occurring after the original request is returned. This ends up providing incomplete data to the content template.
Additional information concerning my build setup:
npm#2.14.2
node#4.0.0
webpack#1.12.2
babel#5.8.34
babel-loader#5.4.0
I believe the issue is in your forEach call in _scan. For reference, see this passage in Taming the asynchronous beast with ES7:
However, if you try to use an async function, then you will get a more subtle bug:
let docs = [{}, {}, {}];
// WARNING: this won't work
docs.forEach(async function (doc, i) {
await db.post(doc);
console.log(i);
});
console.log('main loop done');
This will compile, but the problem is that this will print out:
main loop done
0
1
2
What's happening is that the main function is exiting early, because the await is actually in the sub-function. Furthermore, this will execute each promise concurrently, which is not what we intended.
The lesson is: be careful when you have any function inside your async function. The await will only pause its parent function, so check that it's doing what you actually think it's doing.
So each iteration of the forEach call is running concurrently; they're not executing one at a time. As soon as the one that matches the criteria i === keys.length - 1 finishes, the promise is resolved and _scan returns, even though other async functions called via forEach are still executing.
You would need to either change the forEach to a map to return an array of promises, which you can then await* from _scan (if you want to execute them all concurrently and then call something when they're all done), or execute them one-at-a-time if you want them to execute in sequence.
As a side note, if I'm reading them right, some of your async functions can be simplified a bit; remember that, while awaiting an async function call returns a value, simply calling it returns another promise, and returning a value from an async function is the same as returning a promise that resolves to that value in a non-async function. So, for example, _get can be:
async _get(type, id) {
var data = _json[id] = _json[id] || await this._api(type, id);
console.log(data)
if (isAsset(data)) {
return data;
} else if (isEntry(data)) {
await this._scan(data);
return data;
} else {
const error = 'Response is not entry/asset.';
console.log(error);
throw error;
}
}
Similarly, _scan could be (assuming you want the forEach bodies to execute concurrently):
async _scan(data) {
if (data && data.fields) {
const keys = Object.keys(data.fields);
const promises = keys.map(async (key, i) => {
var val = data.fields[key];
if (isLink(val)) {
var child = await this._get(val.sys.linkType.toUpperCase(), val.sys.id);
this._inject(data.fields, key, undefined, child);
} else if (isLinkArray(val)) {
var children = await* val.map(async (link) => await this._get(link.sys.linkType.toUpperCase(), link.sys.id));
children.forEach((child, index) => {
this._inject(data.fields, key, index, child);
});
} else {
await new Promise((resolve) => setTimeout(resolve, 0));
}
});
await* promises;
} else {
const error = 'Required data is unavailable.';
console.log(error);
throw error;
}
}

Resources