Https Request Response arriving after my Express API Response - node.js

Question

I am writing an API endpoint which searches Google Book's API to add book titles to the notebooks in my MongoDB Database based on their Id's. I have it working properly, but sometimes my endpoint's response array returns missing the last item. My console logs seem to indicate my endpoint response is being sent before the Https request has finished fetching all the data. I need to solve the endpoint sometimes returning an incomplete array. I think I may need to include an async await somewhere but I'm not sure where. Any help is much appreciated!
Here is my code:
//GET a user's books with notes
const getNotebooks = async (req, res) => {
...
const books = await Book.find({ userId }).sort({createdAt: -1}) //gets all notebooks as array
let results = []
for (let i = 0; i < books.length; i++){
console.log(i + " - " + books[i].gId)
const item = {
id: books[i].userId,
gId: books[i].gId,
title: '',
}
const request = https.request(apiUrl + books[i].gId, async (response) => {
let data = '';
response.on('data', (chunk) => {
data = data + chunk.toString();
});
response.on('end', () => {
const body = JSON.parse(data);
item.title = body.volumeInfo.title
results.push(item)
if (i == books.length - 1){
res.status(200).json(results)
}
});
})
request.on('error', (error) => {
console.log('An error', error);
res.status(400).json({error: error})
});
request.end()
}}

Answer 1

You can use async-await with Promises only, since Node's core https module does not have any build-in promise support, you will have to convert it first into the promise format, then you can use async-await with it, i am not familiar with the standard https module but i will make an example with your code
const getNotebooks = async (req, res) => {
//...
const books = await Book.find({ userId }).sort({createdAt: -1}) //gets all notebooks as array
let results = []
for (let i = 0; i < books.length; i++){
console.log(i + " - " + books[i].gId)
const item = {
id: books[i].userId,
gId: books[i].gId,
title: '',
}
const options = apiUrl + books[i].gId
// now you can use await to this function which return a promise
await makeRequest(options, res, item, results, books )
}
}
// Making this function into a promise outside of the scope from your getNotebooks function.
function makeRequest(options, res, item, results, books) {
return new Promise((resolve, rejects) => {
const request = https.request(options, (response) => {
let data = '';
response.on('data', (chunk) => {
data = data + chunk.toString();
});
response.on('end', () => {
const body = JSON.parse(data);
resolve(body);
item.title = body.volumeInfo.title
results.push(item)
if (i == books.length - 1){
res.status(200).json(results)
}
});
});
request.on('error', (error) => {
console.log('An error', error);
rejects(error);
res.status(400).json({error: error})
});
request.end()
});
};
Alternatively you can also use a more modern way to request data, for example with fetch which is now available in node, or even Axios library, if you don't want any dependency, fetch should be the way to go, an example if there is no mistake and assuming is a GET method request:
const getNotebooks = async (req, res) => {
//...
try {
const books = await Book.find({ userId }).sort({createdAt: -1}) //gets all notebooks as array
let results = []
for (let i = 0; i < books.length; i++) {
console.log(i + " - " + books[i].gId)
const item = {
id: books[i].userId,
gId: books[i].gId,
title: '',
}
// using fetch on node js is now possible without any package to install
const response = await fetch(apiUrl + books[i].gId)
const data = await response.json();
item.title = data.volumeInfo.title
results.push(item)
if (i == books.length - 1){
res.status(200).json(results)
}
}
} catch (error) {
res.status(400).json({error: error})
};
};
As you can see you would write 2 line code instead of 20+ line.