Limiting amount of async BigQuery Jobs running on express server - node.js

Question

I have an express server that is pulling data from Google BigQuery. An array of objects is provided. I want to pull sales data for each store in a district, but the table holds sales information based only on store and doesn't have district information. I was accomplishing this by sending one query per district, but once the array has more than 50 district I get errors. The results are stored in individual CSV files by district. So it is convenient to send individual queries and dump it into the CSVs. BigQuery only allows 50 jobs at a given time. I am looking for the best way to adapt the below code to call asyncQuery(query) 50 times then only make the next call when a previous call has returned. I have been trying to work the job status using the job.getMetadata() but no luck yet.
Thanks for any help you can offer
const array = [{
district: "north"
store: "1001,1002"
},
{
district: "south"
store: "1003"
},{
district: "west"
store: "1004"
}
]
function apiCall(array) {
array.forEach(element => {
let stores = element.store.toString()
let query = `SELECT store, sku, tot_sales, price
FROM big-query-table
WHERE
store IN (${stores})`
asyncQuery(query)
.then(resp => {
console.log(resp)
}).catch(err => {
console.error('ERROR:', err);
})
})
return "Running Jobs"
}
function asyncQuery(sqlQuery) {
const options = {
query: sqlQuery,
useLegacySql: false,
};
let job;
return bigquery
.createQueryJob(options)
.then(results => {
job = results[0];
console.log(`Job ${job.id} started.`);
return job.promise();
})
.then(() => {
// Get the job's status
return job.getMetadata();
})
.then(metadata => {
// Check the job's status for errors
const errors = metadata[0].status.errors;
if (errors && errors.length > 0) {
throw errors;
}
})
.then(() => {
console.log(`Job ${job.id} completed.`);
return job.getQueryResults();
})
.then(results => {
const rows = results[0];
return rows;
})
.catch(err => {
console.error('ERROR:', err);
});
}

Answer 1

With BigQuery - and any other columnar analytical database - you really want to avoid doing 50 queries like:
[*50] SELECT * FROM big-query-table
WHERE storeNumber = ${StoreNumber}
Instead, the best you could do is one query, specifying the columns you are looking for, and all the ids you're looking for:
SELECT col1, col2, col3
FROM big-query-table
WHERE storeNumber IN ('id1', 'id2', ..., 'id50')
Or a straight join:
SELECT col1, col2, col3
FROM big-query-table
WHERE storeNumber IN (SELECT store_id FROM `table`)
Then you won't need to send 50 concurrent queries, and you'll get result in less time and lower cost.