Nodejs - Need to perform around 100 API Async request - node.js

Question

I am new for nodejs and I trying to perform around 100 API request by using axios npm in single request. What will be best code logic to capture all response in better way of performance / error tracking.
Need to capture all response in single JSON file, so I decided to use createWriteStream('filename.json')
to avoid the memory issue.
I tried something
const axios = require('axios');
const fs = require('fs');
const config = require('./config/secret.json');
app.get('/json', (req,res) => {
const linkArr = ['https://apirequest1.com','https://apirequest2.com','https://apirequest3.com','https://apirequest4.com', '...'];
const wipArr = [];
for(let getAPI of linkArr){
axios({
method: 'get',
url: getAPI,
auth: {username: config.username, password: config.secret}
})
.then(function (response){
const writeStream = fs.createWriteStream('wip.json');
writeStream.write(JSON.stringify(response.data));
})
.catch(function (error){
console.log(error);
})
}
res.send('successfully saved all response');
});
Capture all API response in single hit and save them in array after completing the API request, need to write all response in JSON file.
Thanks in advance!

Answer 1

The first issue you have is that you create the stream everytime. This will overwrite the contents each time the promise is resolved. Remove this line.
const writeStream = fs.createWriteStream('wip.json');
You will have something like this.
const axios = require('axios');
const fs = require('fs');
const config = require('./config/secret.json');
const writeStream = fs.createWriteStream('wip.json');
app.get('/json', (req,res) => {
const linkArr = ['https://apirequest1.com','https://apirequest2.com','https://apirequest3.com','https://apirequest4.com', '...'];
const wipArr = [];
for(let getAPI of linkArr){
axios({
method: 'get',
url: getAPI,
auth: {username: config.username, password: config.secret}
})
.then(function (response){
//const writeStream = fs.createWriteStream('wip.json'); // remove this line because it will overwrite the file for each response.
writeStream.write(JSON.stringify(response.data));
})
.catch(function (error){
console.log(error);
})
}
res.send('successfully saved all response');
})
;
EDIT: To wait for all requests, You can try something like this.
app.get('/json', async (req, res) => {
let resp = null;
const writeStream = fs.createWriteStream('wip.json');
const linkArr = ['https://apirequest1.com', 'https://apirequest2.com', 'https://apirequest3.com', 'https://apirequest4.com', '...'];
const promises = [];
for (let getAPI of linkArr) {
promises.push(makeCall(getAPI));
resp = await Promise.all(promises); // resp is array of responses
// for (let i = 0; i < resp.length; i++) {
// writeStream.write(JSON.stringify(resp[i], null, 4)); // to //format the json string
// }
}
for (let i = 0; i < resp.length; i++) {
writeStream.write(JSON.stringify(resp[i], null, 4)); // to format the json string
}
res.send('successfully saved all response');
});
function makeCall(getAPI) {
axios({
method: 'get',
url: getAPI,
auth: { username: config.username, password: config.secret }
})
.then(function(response) {
return response.data;
});
}
I have not tested it but something along those lines. This will run all the requests.
To format JSON strings you can use.
JSON.stringify(resp[i], null, 4).
Have a look at https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify
Edit: The problem was that the writeStream.write(JSON.stringify(resp[i], null, 4)); was inside the loop. Moved it outside.
Added code without testing. This should work for you.
app.get('/json', async(req, res) => {
const writeStream = fs.createWriteStream('wip.json');
const linkArr = ['https://apirequest1.com', 'https://apirequest2.com', 'https://apirequest3.com', 'https://apirequest4.com', '...'];
const promises = [];
for (let getAPI of linkArr) {
promises.push(makeCall(getAPI));
}
const resp = await Promise.all(promises); // resp is array of responses
for (let i = 0; i < resp.length; i++) {
writeStream.write(JSON.stringify(resp[i], null, 4)); // to format the json string
}
res.send('successfully saved all response');
});
function makeCall(getAPI) {
return axios({
method: 'get',
url: getAPI,
auth: { username: config.username, password: config.secret }
})
}