Chaining GET requests to WP REST API in Express - node.js

Question

I am struggling to understand callbacks, promises, and async/await.
What I want to do is read a .csv file inside my project folder that contains 150+ post ID's.
For each one of those ID's I want to make a https GET request to fetch a JSON response from my Wordpress website.
Then for each one of those posts that gets returned I want to insert them in my Firestore database.
I'm struggling with how to properly set up the callback functions.
Please help.
Thanks in advance.
const express = require('express');
const router = express.Router();
const https = require("https");
const Recipe = require("../includes/newrecipe");
var admin = require('firebase-admin');
var serviceAccount = require("../service_key.json");
const collectionKey = "recipes"; //name of the collection
admin.initializeApp({
credential: admin.credential.cert(serviceAccount),
databaseURL: "<MY_FIRESTORE_URL>"
});
const firestore = admin.firestore();
const fs = require('fs');
const parse = require('csv-parser');
function prepareCsvData() {
return new Promise((resolve, reject) => {
//establish empty csvData array and filename to be referenced
var csvData = [];
var filename = 'wprm_recipe_ids.csv';
//read the csv file and push the data object into the array
fs.createReadStream(filename)
.pipe(parse(['ID']))
.on('data', (data) => csvData.push(data))
.on('end', () => { resolve(csvData); });
});
}
function getRecipeFromBlog(recipeId) {
return new Promise((resolve, reject) => {
//make the get request to my website to get the recipe
https.get('<MY_WEBSITE_URL>' + recipeId, (response) => {
var body = "";
response.on('data', function (chunk) { body += chunk; });
response.on('end', () => {
var { recipe } = JSON.parse(body);
//build new recipe to be exported
var newRecipe = new Recipe(recipe);
resolve(newRecipe);
});
});
});
}
/* GET recipes. */
router.get('/', async (req, res, next) => {
//first prepare the csv data
//function returns a promise with the csv data
//that I can then use in the next step
const csvData = await prepareCsvData();
for (var i = 0; csvData.length < i; i++) {
getRecipeFromBlog(csvData[i].ID)
.then((newRecipe) => {
//when I have a recipe for a given recipe ID
//update database in firestore
firestore
.collection(collectionKey)
.doc(""+newRecipe.id)
.set(newRecipe)
.then(function() {
console.log('document written');
});
});
}
res.send('done');
});

Answer 1

You need to do something like below:
Play around this, You'll get it working hopefully!
Let me know if that worked!
router.get("/", async (req, res, next) => {
const csvData = await prepareCsvData();
const recipePromises = [];
// check if data is empty or not
if (!csvData.length) {
return res.send("Empty data");
}
csvData.forEach((element) => {
recipePromises.push(getRecipeFromBlog(element.id));
});
// await for all promises parallelly.
const result = await Promise.all(recipePromises);
// Get a new write batch
const batch = db.batch();
result.forEach((recipe) => {
const ref = db.collection("recipes").doc(`${recipe.id}`);
batch.set(ref, recipe);
});
// Commit the batch
await batch.commit();
res.send("done");
});

Answer 2

The OP code looks pretty close to working. Have the promise-returning functions been tested? Assuming they work, first decorate them as async...
async function prepareCsvData() {...}
async function getRecipeFromBlog(recipeId) {...}
Create another promise-returning function to insert many recipes into firebase...
async function writeRecipesToFB(recipes) {
const collectionRef = collection(collectionKey);
const promises = recipes.map(recipe => {
return collectionRef.doc(`${recipe.id}`).set(recipe);
});
return Promise.all(promises)
}
As another answer suggests, as an alternative, firebase's batch write is a good idea...
async function writeRecipesToFB(recipes) {
// as a set of promises
const collectionRef = collection(collectionKey);
const batch = db.batch();
recipes.forEach(recipe => {
const docRef = collectionRef.doc(`${recipe.id}`)
batch.set(docRef, recipe)
});
return batch.commit();
}
Now the express function is easy to write...
router.get('/', async (req, res, next) => {
const csvData = await prepareCsvData();
let promises = csvData.map(row => {
return getRecipeFromBlog(row.ID);
});
const recipes = await Promise.all(promises);
await writeRecipesToFB(recipes);
res.send('done');
});