Running concurrent firebase requests using promises - node.js

I am trying to execute number of Firebase Cloud Storage queries concurrently using Promise.all() :
exports.onNewOrder =
functions.database.ref('/orders/{orderId}').onCreate(event => {
const data = event.val();
const orderedBy = data.ordered_by;
const creations = data.creations;
var promises = []
for (var entry of creations.entries()) {
var key = entry[0], value = entry[1];
var creationPromise = admin.database().ref("creations/"+ key);
creationPromise.once("value")
.then((crSnapshot)=>{
const crName = crSnapshot.val().name;
const created = crSnapshot.val().created;
return "crName+'\n'+created”;
}).catch((error) => {
console.log("creation details error : "+ error);
});
promises.push(creationPromise);
}
const userPromise = admin.database().ref("users/"+ orderedBy);
userPromise.once("value")
.then((snapshot) => {
var name = snapshot.val().name;
var email = snapshot.val().email;
const userDetails = name+'\n'+email;
return userDetails;
}).catch((error) => {
console.log("user details error :"+ error);
});
promises.push(userPromise);
return Promise.all(promises).then(results => {
console.log("results: " + results)
return true;
}).catch((error) => {
console.log("sendMail failed : "+ error);
});
});
I am expecting to see complete query result for every executed promise, however I am getting the following instead:
results:
https://....firebaseio.com/creations/cr_1bfd5f16212f44c792a2cbe3e9c6e4cc
https://....firebaseio.com/creations/cr_343c363c2a214487ba8cb7101653f6af
https://....firebaseio.com/creations/cr_e3f7a0bafd1542ac898d96e86155b94e
https://....firebaseio.com/users/us_5e4c28e18e63454c861caa5c4fe6a6f0
Am I missing something?

The following should do the trick.
As detailed here:
The Promise.all(iterable) method returns a single Promise that
resolves when all of the promises in the iterable argument have
resolved or when the iterable argument contains no promises. It
rejects with the reason of the first promise that rejects.
Therefore you need to add some promises to the promises array.
The once() method does return a promise, as explained here in the doc.
exports.onNewOrder =
functions.database.ref('/orders/{orderId}').onCreate(event => {
const data = event.val();
const orderedBy = data.ordered_by;
const creations = data.creations;
const promises = []
for (var entry of creations.entries()) {
var key = entry[0];
var creationPromise = admin.database().ref("creations/"+ key).once("value");
promises.push(creationPromise);
}
const userPromise = admin.database().ref("users/"+ orderedBy).once("value");
promises.push(userPromise);
return Promise.all(promises)
.then(results => {
console.log("results: " + results);
return true;
//actually here you will probably do something else than just write the results to the console.
//In this case you should return a promise (instead of true).
}).catch((error) => {
console.log("sendMail failed : "+ error);
return false;
});
});

Related

Waiting for async function in for loop

I need to wait for an async method (a call to my database) for every object in an array. Right now I have a for loop going through the array calling the async method on each object. The async function is successful but I need to wait for every async call to finish before moving on. Doing some research I have found that Promises combined with await or other methods are the solution to my problem but I haven't been able to figure it out. Here is my code I have right now.
Here is my class with the async function
Vacation : class Vacation {
constructor(id, destination, description, attendee_creator_id) {
this.id = id;
this.destination = destination;
this.description = description;
this.attendee_creator_id = attendee_creator_id;
this.creator = undefined;
this.votes = undefined;
}
find_creator(pool){
return new Promise(function (resolve, reject) {
var self = this;
var query = "SELECT * FROM vacation_attendee WHERE id = " + self.attendee_creator_id;
pool.query(query, function(error, result) {
if (error) {
console.log("Error in query for vacation creator " + error);
return reject(error);
}
var creator = new attendee.VacationAttendee(result.rows[0].full_name, result.rows[0].email, result.rows[0].password_hash);
self.creator = creator;
console.log("creator found ----> " + self.creator.full_name + "in " + self.destination);
resolve(true);
});
})
}
here is how i'm calling the async function
function get_all_vacations(callback) {
var sql_vacations_query = "SELECT * FROM vacation";
pool.query(sql_vacations_query, function(error, vacations_query_result) {
if (error) {
console.log("error in vacations query " + error);
callback(error);
}
var all_complete = loop_through_vacations(vacations_query_result.rows, pool);
callback(null, all_complete);
});
}
async function loop_through_vacations(vacations_incomplete, pool) {
var all_vacations = [];
for (var vacation_data of vacations_incomplete) {
var vacation_at_index = new vac.Vacation(vacation_data.id, vacation_data.destination, vacation_data.description, vacation_data.attendee_creator_id);
vacation_at_index.find_creator(pool)
.then(()=> {
all_vacations.push(vacation_at_index);
})
.catch(error=> {
console.log(error);
});
console.log("passed vacation " + vacation_at_index.destination);
}
return await Promise.all(all_vacations);
}
You do it in a wrong way, you don't wait anything in you for-loop.
return await Promise.all(all_vacations); does not work, because all_vacations is not a Promise array.
In your case, we have many way to do this, my way just is a example: Create a array to store all promises what have been created in your for-loop, then wait until the all promises finished by Promise.all syntax.
async function loop_through_vacations(vacations_incomplete, pool) {
var all_vacations = [];
var promises = []; // store all promise
for (var vacation_data of vacations_incomplete) {
var vacation_at_index = new vac.Vacation(vacation_data.id, vacation_data.destination, vacation_data.description, vacation_data.attendee_creator_id);
promises.push( // push your promise to a store - promises
vacation_at_index.find_creator(pool)
.then(() => {
all_vacations.push(vacation_at_index)
})
.catch((err) => {
console.log(err); // Skip error???
})
);
}
await Promise.all(promises); // wait until all promises finished
return all_vacations;
}
I don't know why you use async/await mix with callback style, I recommend async/await for any case:
Mix style:
function get_all_vacations(callback) {
var sql_vacations_query = "SELECT * FROM vacation";
pool.query(sql_vacations_query, async function(error, vacations_query_result) { // async
if (error) {
console.log("error in vacations query " + error);
return callback(error); // return to break process
}
var all_complete = await loop_through_vacations(vacations_query_result.rows, pool); // await
callback(null, all_complete);
});
}
async/await:
async function get_all_vacations() {
var sql_vacations_query = "SELECT * FROM vacation";
var rows = await new Promise((resolve, reject) => {
pool.query(sql_vacations_query, function(error, vacations_query_result) {
if (error) {
console.log("error in vacations query " + error);
return reject(error);
}
resolve(vacations_query_result.rows);
});
})
var all_complete = await loop_through_vacations(rows, pool); // await
return all_complete;
}
For me, you can try to reformat your SQL queries, the code will be cleaner and it will take less time to make a big query once than multiple queries.
If I understand joins in your database:
SELECT *
FROM vacation_attendee AS attendee
INNER JOIN vacation ON vacation.attendee_creator_id = attendee.id
-- WHERECLAUSEYOUWANT
Then:
async function get_all_vacations(callback){
const allComplete = await makeThatBigQuery(); // + treat data as you want
callback(null, allComplete);
}
In your definition of get_all_vacations you don't wait all_complete, so the callback is called before all_complete is ready
This is how the await/promises part of the code should look like:
function get_all_vacations(callback) {
var sql_vacations_query = "SELECT * FROM vacation";
pool.query(sql_vacations_query, async function(error, vacations_query_result) {
if (error) {
console.log("error in vacations query " + error);
callback(error);
}
var all_complete = await loop_through_vacations(vacations_query_result.rows, pool);
callback(null, all_complete);
});
}
async function loop_through_vacations(vacations_incomplete, pool) {
const promises = [];
for (var vacation_data of vacations_incomplete) {
var vacation_at_index = new vac.Vacation(vacation_data.id, vacation_data.destination, vacation_data.description, vacation_data.attendee_creator_id);
promises.push(vacation_at_index.find_creator(pool));
}
return await Promise.all(promises);
}
Ideally you should be removing all the callbacks(not sure if asyn/await is supported for pool.query methods) and change the whole code to async/await style. Also you should be resolving the promise from 'find_creator' with resolve(creator), so that the value is obtained when the promise is resolved.

Handling promises inside the forEach loop

I am trying to run a series of tasks. Each task is dynamic, and could have different rules to follow. This will be executed on AWS-Lambda.
I have an array of JSON. It has a body with task name in it, and it also has attributes.
I need to dynamically load a javascript file with the name inside the body.
I need to wait until all is finished inside that task. Or it failed (regardless where). If the fail happens, I will need to write that data inside the current record inside the forEach loop.
I have old issue, where my forEach is finished first without waiting for the task to complete.
This is the forEach loop:
const jobLoader = require('./Helpers/jobLoader');
event.Records.forEach(record => {
const { body: jobName } = record;
const { messageAttributes } = record;
const job = jobLoader.loadJob(jobName);
job.runJob(messageAttributes).then(res => {
console.log('Show results');
return; // resume another record from forEach
}).catch(err => {
record.failed = true;
record.failureMessage = err.message;
console.log('I errored');
});
console.log('All Done');
});
The problem is that message All Done is printed, and then the message show results is printed. I get results from the database once it comes for execution.
This is the file that loads a task:
exports.loadJob = (jobName) => {
const job = require(`../Tasks/${jobName}`);
return job;
};
This is the file that contains actual task:
const mySqlConnector = require('../Storage/mySql');
exports.runJob = async (params) => {
let payload = {};
let dataToSend = await getUserName(params.userId.stringValue);
payload.dataToSend = dataToSend;
let moreDataToSend = await getEvenMoreData(params.userId.stringValue);
payload.moreDataToSend = moreDataToSend;
return await sendData(payload);
};
const getUserName = async (userId) => {
const query = 'SELECT * FROM user_data';
return await mySqlConnector.handler(query);
};
const getEvenMoreData = async (userId) => {
const query = 'SELECT * FROM user_data';
return await mySqlConnector.handler(query);
};
const sendData = (payload) => {
//this should be Axios sending data
};
And this is the mySql connector itself:
const mysql = require('promise-mysql');
exports.handler = async (query) => {
return mysql.createConnection({
host : '127.0.0.1',
user : 'root',
password : '',
database: 'crm'
}).then(conn =>{
let result = conn.query(query);
conn.end();
return result;
}).then(rows => {
//console.log("These are rows:" + rows);
return rows;
}).catch(error => {
return error;
});
};
The task file can have any number of things it needs to complete, which will be different when I start adding tasks.
I need that job.runJob completes, or that it catches an error, from whatever location it originated, so I can continue with the forEach.
I have tried using map and what not, but the end result is always the same.
What am I doing wrong?
You can use Promise.all method :
const promises = event.Records.map(record => {
const { body: jobName } = record;
const { messageAttributes } = record;
const job = jobLoader.loadJob(jobName);
return job.runJob(messageAttributes).then(res => {
console.log('Show results', res);
}).catch(err => {
record.failed = true;
record.failureMessage = err.message;
console.log('I errored');
throw new Error('Your error !');
});
});
try {
const results = await Promise.all(promises);
console.log('All done');
} catch (e) {
console.log('Something has an error', e);
}
don't forget to make your function async !
I managed to solve it, and still keep details about the execution:
Something like this:
for (let prop in event.Records){
const { body: jobName } = event.Records[prop];
const { messageAttributes } = event.Records[prop];
const job = jobLoader.loadJob(jobName);
await job.runJob(messageAttributes).then(res => {
console.log('Show results', res);
}).catch(err => {
event.Records[prop].failed = true;
event.Records[prop].failed = err.message;
console.log('I errored');
});
}

Use async forEach loop while fetching data from firestore

I have firestore data somewhat like this:
"Support": {
"userid":"abcdxyz",
"message": "hello"
}
I am using nodejs to fetch my data and I also want to show the email address and name of the person who sent this message. So I am using following function:
database.collection("support").get().then(async function (collections) {
var data = [];
console.log("data collected");
collections.forEach(async function (collection) {
var temp = {};
var collectionData = collection.data()
var userInfo = await getUserDetails(collectionData.userId)
temp.name = userInfo.name
temp.supportMessage = collectionData.supportMessage
data.push(temp)
console.log("data pushed")
});
console.log("data posted")
return res.status(200).end(JSON.stringify({ status: 200, message: "Support Message fetched successfully.", data: data }))
}).catch(error => {
return res.status(500).end(JSON.stringify({ status: 500, message: "Error: " + error }))
});
Here the sequence of logs is following: data collected, data posted, data pushed
I want the sequence like this: data collected, data pushed (x times), data posted
Use following code:
database.collection("support").get().then(async function (collections) {
var data = [];
console.log("data collected");
for await(let collection of collections){
var temp = {};
var collectionData = collection.data()
var userInfo = await getUserDetails(collectionData.userId)
temp.name = userInfo.name
temp.supportMessage = collectionData.supportMessage
data.push(temp)
console.log("data pushed")
}
console.log("data posted")
return res.status(200).end(JSON.stringify({ status: 200, message: "Support Message fetched successfully.", data: data }))
}).catch(error => {
return res.status(500).end(JSON.stringify({ status: 500, message: "Error: " + error }))
});
OR
Use can use
var promise = Promise.all(collections.map((collection) =>{
...
return await ... //or a promise
}));
promise.then(() => {
console.log("posted");
return res.status(200).end(...);
})
I solved my answer with the help of #estus comment.
Credit: #estus
var data = [];
var tempCollection = [];
collections.forEach(collection => {
tempCollection.push(collection.data());
});
for (collection of tempCollection) {
var temp = {};
var userInfo = await getUserDetails(collection.userId)
temp.name = userInfo.name
temp.supportMessage = collection.supportMessage
data.push(temp)
}
It solved my problem very easily.
I know this might not the OP's exact use case but if you're interested in compiling the results of a collection query into an array, you can still use the .docs property of a QuerySnapshot to obtain the list of items:
...
const userId = <some-id>;
const usersRef = db.collection(`users`)
const usersSnapshot = await usersRef.where("id", "==", userId).get()
if (usersSnapshot.empty) {
console.log('found no matching user ', userId);
return;
}
console.log(`found ${usersSnapshot.size} user records`);
const userResults = []
// this block here doesn't construct the results array before the return statement
// because .foreach enumerator doesn't await: https://stackoverflow.com/q/37576685/1145905
// usersSnapshot.forEach((doc) => {
// // doc.data() is never undefined for query doc snapshots
// //console.log(doc.id, " => ", doc.data());
// userResults.push[doc.data()];
// });
for (user of usersSnapshot.docs) {
// console.log(user.id, " => ", user.data());
userResults.push(user.data());
}
...
A more detailed example is here

async function in node only runs after response sent

I have the method:
const getTaskOrCategoryRecursive = async (type, id)=>{
const rslt = await type.findOne.call(type, {_id:id},{}, standardOptions, err=>{
if(err){
return({err: err});
}
});
const result = rslt.toObject();
const tasks = result.tasks;
const events = result.events;
result.children = {};
result.children.tasks = tasks.map(async taskId=>{
const taskIdString = taskId.toString();
const rtrn = await getTaskRecursive(taskIdString, err=>{
if(err){
return({err: err});
}
});
return rtrn;
});
result.children.events = events.map(async eventId=>{
const eventIdString = eventId.toString();
await Event.findOne({_id:eventIdString}, standardOptions,err=>{
if(err){
return({err: err});
}
});
});
return result;
}
It's called by two methods:
const getTaskRecursive = (taskId)=>{
return getTaskOrCategoryRecursive(Task, taskId)
}
and
const getCategoryRecursive=(categoryId)=>{
return getTaskOrCategoryRecursive(Category,categoryId);
};
which are called by the function
router.get('/:id', verifyToken, async (req,res)=>{
Branch.getCategoryRecursive(req.params.id)
.then(
(cat)=>{
res.status(200).send(cat);
},
(err)=>{
res.status(500).send(err);
}
)
});
When I try to run the program, first the getCategoryRecursive method runs, then res.status(200).send(cat); then the getTasksRecursive method which gets the children of the object being sent in the response. getTasksRecursive does what it is supposed to, but it's running after the response is sent so the object is empty.
How do I make my asynchronous method run before the response is sent?
UPDATE: Using Aritra Chakraborty's answer, I changed it to the following and it worked.
First I separated the .map into a new function:
const getAllTasksRecursive = async(taskIds)=>{
const rtrn = await Promise.all(
taskIds.map(
async taskId=>{
return await getTaskRecursive(taskId.toString(), err=>{if(err){return {err:err}}});
}
)
)
return rtrn;
}
Then I called it in the previous function using:
result.children.tasks = await getAllTasksRecursive(tasks);
Now I am getting the data back in the response.
That's because internally a map or foreach can look something like this:
Array.prototype.forEach = function (callback) {
// this represents our array
for (let index = 0; index < this.length; index++) {
// We call the callback for each entry
callback(this[index], index, this)
}
}
It will call the callback alright, but it will not wait for the callback to complete before running the next one.
So, you need one async foreach of some sort,
async function asyncForEach(array, callback) {
for (let index = 0; index < array.length; index++) {
await callback(array[index], index, array)
}
}
Note, how the callback is being awaited.
Your code will have to be something like this:
const start = async () => {
await asyncForEach([1, 2, 3], async (num) => {
await waitFor(50)
console.log(num)
})
console.log('Done')
}
start()
Refer: https://codeburst.io/javascript-async-await-with-foreach-b6ba62bbf404 This article helped me alot in the past.

Getting empty array in nodejs

I am posting value But I am getting empty array . I know its node asynchronous problem . But I don't know how do i solve this. I have refer this following link:
How do I return the response from an asynchronous call?
But I could not able to understand . Kindly help me to understand promises and how do i use that in my code.
router.post('/inspection_list', function (req, res) {
var id = req.body.project_id;
console.log(id)
// res.send("ok")
db.inspection.findOne({'_id':id},(err,response)=>{
if(err){
console.log("error");
}
else{
console.log("Data")
var inspection = [];
var data = response.inspection_data;
var f = data.map(function (item) {
var fielduser = item.fielduser_id
db.fielduser.findOne({'_id': mongoose.Types.ObjectId(fielduser)},(err,user)=>{
console.log(user.owner_name);
console.log(item.inspection_name)
inspection.push({inspection_name:item.inspection_name,field_user_name : user.owner_name})
})
});
console.log(inspection) // Here am getting empty value
// setTimeout(function(){ console.log(inspection) }, 5000); my timeout code
}
})
});
router.post('/inspection_list', async function (req, res) {
var id = req.body.project_id;
try{
var response = await db.inspection.findOne({'_id':id})
var inspection = [];
var data = response.inspection_data;
for ( var i = 0; i<data.length; i++){
var item = data[i]
var fielduser = item.fielduser_id
var user = await db.fielduser.findOne({'_id': mongoose.Types.ObjectId(fielduser)})
inspection.push({inspection_name:item.inspection_name,field_user_name : user.owner_name})
}
}
catch(err){
throw err
}
})
This uses async and await, you can use it if you are using node version >=7.6
Also note the following:
router.post('/inspection_list', async function (req, res)
Handling each error seperately
router.post('/inspection_list', async function (req, res) {
var id = req.body.project_id;
try{
var response = await db.inspection.findOne({'_id':id})
}
catch(err){
// handle error here
throw err
}
var inspection = [];
var data = response.inspection_data;
for ( var i = 0; i<data.length; var item = data[i]
var fielduser = item.fielduser_id
try{
var user = await db.fielduser.findOne({'_id': mongoose.Types.ObjectId(fielduser)})
}
catch(err){
// handle error
}
inspection.push({inspection_name:item.inspection_name,field_user_name : user.owner_name})
}
})
Using mongoose would be the easy way out, it returns Promises for all query and save functions, so you'd simply do:
YourModel.findOne({params}).then(() => {...})
If you're unable to do that, in your case, a 'promisified' example would be:
var findAndFillArray = (project_id) => new Promise((resolve) => {
.... your previous code here ....
inspection.push({inspection_name:item.inspection_name,field_user_name :
user.owner_name})
if (data.length === inspection.length){ // Or some other preferred condition
resolve(inspection);
}
})
Then you'd call this function after you get the id, like any other function:
var id = req.body.project_id;
findAndFillArray(id).then((inspection_array) => {
res.send(inspection_array) // Or whatever
})
Now, map and all list functions are synchronous in JS, are you sure the error is due to that?

Resources