Confused on how to properly use promises in http Firebase Cloud Function - node.js

I am new to using cloud functions for Firebase with an http triggered function and I am confused on how to properly terminate the function. I'm not sure if I should be using res.sendStatus, returning a promise, or both.
The goal of my function is to loop through several documents in the collection 'communities'. Each community has a collection of documents where I query the document with the highest value of 'hotScore'. I then send an iOS push notification containing that document to a topic (all users in that given community).
Unfortunately, I am getting several errors when the code is run such as Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client and Unhandled rejection. I'm pretty such this is due to my negligence in handling function termination, although I have been confused by the online resources I have looked at so far. Would someone mind taking a look at my code/pointing me in the right direction? Thank you so much!
exports.sendNotificationTrendingPost = functions.https.onRequest(async (req, res) => {
//Get communities collection from Firestore
return admin.firestore().collection('communities').get().then((communities) => {
var communityPromises = [];
//Loop through each community
communities.forEach((community) => {
let communityID = community.get('communityID');
let communityName = community.get('name');
//Get the post with the highest hotScore
let communityPromise = admin.firestore().collection('communities').doc(communityID).collection('posts').orderBy('hotScore', 'desc').limit(1).get().then((posts) => {
let hottestPost = posts[0];
let postID = hottestPost.get('postID');
let postText = hottestPost.get('text');
let currentDate = Date.now() / 1000;
var message;
//Verify that the hottest post was posted in the past 24 hours
if (hottestPost.get('date') > (currentDate - 86400)) {
//Build the notification text (shortening if too long)
let shortenedPostText = postText.substring(0,60);
var textEnd = '';
if (postText.length > 60) {
textEnd = '...';
}
let notificationText = 'Trending post on ' + communityName + ': ' + shortenedPostText + textEnd;
//Build the push notification
message = {
apns: {
headers: {
'apns-push-type': 'alert'
},
payload: {
aps: {
alert: {
body: notificationText,
},
},
postID: postID,
},
},
topic: communityID
}
}
//Send the message and return the promise
if (message === null) {
return null;
} else {
return admin.messaging().send(message);
}
})
.catch(error => {
console.log(error);
res.status(500).send(error);
})
if (communityPromise !== null) {
communityPromises.push(communityPromise);
}
})
res.sendStatus(200);
return Promise.all(communityPromises);
})
.catch(error => {
console.log(error);
res.status(500).send(error);
})
})

As samthecodingman advised, it is much better to use async/await in your case, as it will simplify the code and will make it much easier to read.
The following changes should do the trick (untested). Note how we use an Array of Community names in order to pass the names from one loop to the other. This works because, with Promise.all(), the returned values are in order of the Promises passed, regardless of completion order.
exports.sendNotificationTrendingPost = functions.https.onRequest(async (req, res) => {
try {
const db = admin.firestore();
const communitiesQuerySnap = await db.collection('communities').get();
const communityPromises = [];
const communityNames = [];
communitiesQuerySnap.forEach((community) => {
let communityID = community.get('communityID');
let communityName = community.get('name');
communityNames.push(communityName);
communityPromises.push(db.collection('communities').doc(communityID).collection('posts').orderBy('hotScore', 'desc').limit(1).get())
});
const postsQuerySnapArray = await Promise.all(communityPromises);
const messagePromises = [];
postsQuerySnapArray.forEach((postsQuerySnap, index) => {
const hottestPost = postsQuerySnap.docs[0];
const postID = hottestPost.get('postID');
const postText = hottestPost.get('text');
const currentDate = Date.now() / 1000;
let message;
if (hottestPost.get('date') > (currentDate - 86400)) {
//Build the notification text (shortening if too long)
let shortenedPostText = postText.substring(0, 60);
var textEnd = '';
if (postText.length > 60) {
textEnd = '...';
}
const communityName = communityNames[index]; // The two Arrays postsQuerySnapArray and communityName have the same order, because Promise.all keeps the order.
let notificationText = 'Trending post on ' + communityName + ': ' + shortenedPostText + textEnd;
//Build the push notification
message = {
apns: {
headers: {
'apns-push-type': 'alert'
},
payload: {
aps: {
alert: {
body: notificationText,
},
},
postID: postID,
},
},
topic: communityID
}
messagePromises.push(admin.messaging().send(message));
}
})
await Promise.all(messagePromises);
res.status(200).send({ result: "completed" }); // Or res.end()
} catch (error) {
console.log(error);
res.status(500).send(error);
}
});

Related

Https Request Response arriving after my Express API Response

I am writing an API endpoint which searches Google Book's API to add book titles to the notebooks in my MongoDB Database based on their Id's. I have it working properly, but sometimes my endpoint's response array returns missing the last item. My console logs seem to indicate my endpoint response is being sent before the Https request has finished fetching all the data. I need to solve the endpoint sometimes returning an incomplete array. I think I may need to include an async await somewhere but I'm not sure where. Any help is much appreciated!
Here is my code:
//GET a user's books with notes
const getNotebooks = async (req, res) => {
...
const books = await Book.find({ userId }).sort({createdAt: -1}) //gets all notebooks as array
let results = []
for (let i = 0; i < books.length; i++){
console.log(i + " - " + books[i].gId)
const item = {
id: books[i].userId,
gId: books[i].gId,
title: '',
}
const request = https.request(apiUrl + books[i].gId, async (response) => {
let data = '';
response.on('data', (chunk) => {
data = data + chunk.toString();
});
response.on('end', () => {
const body = JSON.parse(data);
item.title = body.volumeInfo.title
results.push(item)
if (i == books.length - 1){
res.status(200).json(results)
}
});
})
request.on('error', (error) => {
console.log('An error', error);
res.status(400).json({error: error})
});
request.end()
}}
You can use async-await with Promises only, since Node's core https module does not have any build-in promise support, you will have to convert it first into the promise format, then you can use async-await with it, i am not familiar with the standard https module but i will make an example with your code
const getNotebooks = async (req, res) => {
//...
const books = await Book.find({ userId }).sort({createdAt: -1}) //gets all notebooks as array
let results = []
for (let i = 0; i < books.length; i++){
console.log(i + " - " + books[i].gId)
const item = {
id: books[i].userId,
gId: books[i].gId,
title: '',
}
const options = apiUrl + books[i].gId
// now you can use await to this function which return a promise
await makeRequest(options, res, item, results, books )
}
}
// Making this function into a promise outside of the scope from your getNotebooks function.
function makeRequest(options, res, item, results, books) {
return new Promise((resolve, rejects) => {
const request = https.request(options, (response) => {
let data = '';
response.on('data', (chunk) => {
data = data + chunk.toString();
});
response.on('end', () => {
const body = JSON.parse(data);
resolve(body);
item.title = body.volumeInfo.title
results.push(item)
if (i == books.length - 1){
res.status(200).json(results)
}
});
});
request.on('error', (error) => {
console.log('An error', error);
rejects(error);
res.status(400).json({error: error})
});
request.end()
});
};
Alternatively you can also use a more modern way to request data, for example with fetch which is now available in node, or even Axios library, if you don't want any dependency, fetch should be the way to go, an example if there is no mistake and assuming is a GET method request:
const getNotebooks = async (req, res) => {
//...
try {
const books = await Book.find({ userId }).sort({createdAt: -1}) //gets all notebooks as array
let results = []
for (let i = 0; i < books.length; i++) {
console.log(i + " - " + books[i].gId)
const item = {
id: books[i].userId,
gId: books[i].gId,
title: '',
}
// using fetch on node js is now possible without any package to install
const response = await fetch(apiUrl + books[i].gId)
const data = await response.json();
item.title = data.volumeInfo.title
results.push(item)
if (i == books.length - 1){
res.status(200).json(results)
}
}
} catch (error) {
res.status(400).json({error: error})
};
};
As you can see you would write 2 line code instead of 20+ line.

When using forEach in a Cloud Function, I can't make .sendToDevice() method work

I can send messages to the iOS device using the second function shown below.
I get the document id in the collection name "users" which is at the first level and send the message using the token stored in the tokens subcollection therefore admin.firestore().collection('users').doc(userId).collection('tokens').
I have to change the way the function looks for the user. Rather than relying on the document id of the user, I now need a query in order to find the user. Being a query, unless I'm wrong, I'm forced to use forEach in order to send the message to the user. The function now looks as shown immediately below. In essence, once I know I have the user that needs to receive the message, I'm using the original function format to send the message but the message is never sent. All I see in the logs is Firebase messaging error and I have yet to figure out where the mistake is.
exports.sendMessage = functions.https.onRequest(async (res, response) => {
const body = res.body;
const orderTotal = body.total;
const orderId = String(body.id);
const query = await usersRef.where('token', '==', token).get();
if (query.empty) {
console.log('No matching documents.');
return;
}
query.forEach(doc => {
const tokens = usersRef.doc(doc.id).collection('tokens');
tokens.get()
.then(snapshot => {
const results = [];
snapshot.forEach(doc => {
const fcmToken = doc.data().fcmToken
console.log("fcmToken =>", fcmToken);
results.push(fcmToken);
})
const payload = {
notification: {
title_loc_key: 'notify_title',
subtitle_loc_key: 'notify_subtitle',
body_loc_key: 'notify_body',
badge: '1',
sound: 'cha-ching.caf',
mutable_content: 'true'
},
data: {
'total': orderTotal,
'orderId': orderId
}
}
response.send([results, , payload])
admin.messaging().sendToDevice(results, payload).then((response) => {
// Response is a message ID string.
console.log('Successfully sent message:', response);
return { success: true };
}).catch((error) => {
return { error: error.code };
})
})
.catch(err => {
console.log("Error getting documents", err);
});
});
});
This is the original function which I used when using the document id.
exports.sendMessage = functions.https.onRequest(async (res, response) => {
const body = res.body
const orderTotal = body.total
const orderId = String(body.id)
const tokenReference = admin.firestore().collection('users').doc(userId).collection('tokens')
const tokenSnapshots = await tokenReference.get()
const results = []
tokenSnapshots.forEach(tokenSnapshot => {
const fcmToken = tokenSnapshot.data().fcmToken
results.push(fcmToken)
})
const payload = {
notification: {
title_loc_key: 'notify_title',
subtitle_loc_key: 'notify_subtitle',
body_loc_key: 'notify_body',
badge: '1',
sound: 'cha-ching.caf',
mutable_content: 'true'
},
data: {
'total': orderTotal,
'orderId': orderId
}
}
response.send([results, , payload])
admin.messaging().sendToDevice(results, payload).then((response) => {
console.log('Successfully sent message:', response);
return { success: true };
}).catch((error) => {
return { error: error.code };
})
})
Screenshot of the error:
The onRequest() function terminates when you return a response. You are using sendToDevice() after response.send(). Also make sure you are handling all the promises correctly. Try refactoring the using async-await syntax as shown below:
exports.sendMessage = functions.https.onRequest(async (res, response) => {
try {
const body = res.body;
const orderTotal = body.total;
const orderId = String(body.id);
const query = await usersRef.where("token", "==", "TOKEN").get();
if (query.empty) {
console.log("No matching documents.");
return;
}
// Query tokens of all users at once
const tokenSnapshots = await Promise.all(
query.docs.map((user) => usersRef.doc(user.id).collection("tokens").get())
);
// Array of all fcmTokens
const results = tokenSnapshots.reduce((acc, snapshot) => {
acc = [...acc, ...snapshot.docs.map((doc) => doc.data().fcmToken)];
return acc;
}, []);
const payload = { ...FCM_PAYLOAD };
const fcmResponse = await getMessaging().sendToDevice(results, payload);
console.log("Successfully sent message:", fcmResponse);
response.send([results, , payload]);
} catch (error) {
console.log(error);
response.json({ error: "An error occured" });
}
});
Also checkout Terminating HTTP Cloud Functions.
After days of working on this, it turns out there wasn't anything wrong with the function. I don't know how VPN works but the fact that I had it enabled on my iPhone was the reason I wasn't getting the notification.
I paused the VPN and the notification was received.

Why the nodejs heap out of memory for creating Excel file with big data?

I am creating an excel file at nodejs end and returning base64 data to reactJS to download the file. At nodejs end, I am using promise all and fetch data from a server in chunks and append data into Excel as
worksheet.addRows(data);
For data around 20-30k, it is working fine but for data like 100k, it shows me an error heap out of memory at nodejs end.
I have increase memory allocate to nodejs also but same error
node --max_old_space_size=5000 app.js
What I am doing wrong any suggestions?
Nodejs
const axios = require('axios');
var excel = require("exceljs");
const workbook = new excel.Workbook();
const worksheet = workbook.addWorksheet("My Sheet");
worksheet.columns = [
{ header: "TicketId", key: "ticketId" },
{ header: "Email", key: 'user_email' },
{ header: "User", key : 'user_name' },
{ header: "Subject", key: "subject" },
...//many more headers
];
exports.getTicketData = async (req, res, next) => {
res.connection.setTimeout(0);
const { body } = req;
const token = body.token;
const organization_id = body.organization_id;
const server = body.server;
const sideFilter = body.sideFilter;
let baseurl = 'url for server end to fetch data';
if (baseurl) {
let data = new Array();
let limit = 3000;
const promises = [];
try {
let count = await getCount(token,limit, organization_id, baseurl, sideFilter);
for(var i = 1;i<=count;i++) {
promises.push(getData(i,limit,organization_id,token, baseurl, sideFilter));
}
await Promise.all(promises).then((results) => {
}).catch((e) => {
throw e;
});
var base64File = await writeExcelAndUpload(workbook);
return res.status(200).json({ file:base64File });
} catch (err) {
return res.status(400).json({ type:'error', msg:'File not generated please contact support staff' });
}
} else {
return res.status(400).json({ type:'error', msg:'please define server name' });
}
};
let getData = (page,limit, organization_id,token, baseurl, sideFilter) =>{
return new Promise((resolve, reject) => {
axios.post(baseurl+`/v2/get-export`, {
page:page,
organization_id:organization_id,
per_page:limit,
filter: "",
sorted:"",
...sideFilter
},{ headers: {"Authorization" : `Bearer ${token}`} }).then(function (response) {
let dataTemp = response.data.data.data.map((t,i)=>{
return {
...t,
name:t.name,
...//many more columns like 70
}
});
worksheet.addRows(dataTemp);
resolve(true);
}).catch(function (error) {
reject(error);
});
});
}
let getCount = (token,limit, organization_id, baseurl, sideFilter) => {
// run an api and return count against limit
}
let writeExcelAndUpload = async (workbook) => {
const fileBuffer = await workbook.xlsx.writeBuffer();
let base64File = Buffer.from(fileBuffer).toString('base64');
base64File = 'data:application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;base64,'+base64File;
return base64File;
}
Client side reactjs
exportLink = () => {
postData ={
...
};
return axios.post(`${baseurl}/api/ticketing/get-ticket`, postData).then(function (response) {
const downloadLink = document.createElement("a");
const fileName = "export.xlsx";
downloadLink.href = response.data.file;
downloadLink.download = fileName;
downloadLink.click();
}).catch(function(error){
throw error;
});
}
Well, it is kinda expected that you may get a heap out of memory when working with such an amount of entries like 100k.
I could suggest you start using pagination, and instead of fetching e.g. 100k of entries at once fetch 1k of entries do what you need with them, then fetch the next 1k of entries repeat until you processed all entries.

How to consume a RESTful API in Node.js

I'm new to Node.js and I'm creating a simple pagination page. The REST API works fine, but consuming it has left me in limbo.
Here is the REST API (other parts have been taken out for brevity)
const data = req.query.pageNo;
const pageNo =
(typeof data === 'undefined' || data < 1) ? 1 : parseInt(req.query.pageNo);
let query = {};
const total = 10;
query.skip = (total * pageNo) - total;
query.limit = total;
try {
const totalCount = await Users.countDocuments();
const pageTotal = Math.ceil(totalCount / total);
const users = await Users.find({}, {}, query);
return res.status(200).json(users);
} catch (error) {
console.log('Error ', error);
return res.status(400).send(error)
};
};
When I return the json with just the 'users' object, like so return res.status(200).json(users); the page renders correctly, but when I pass in other objects like what I have in the code, it fails. This is how I'm consuming the API:
const renderHomepage = (req, res, responseBody) => {
let message = null;
if (!(responseBody instanceof Array)) {
message = 'API lookup error';
responseBody = [];
} else {
if (!responseBody.length) {
message = 'No users found nearby';
}
}
res.render('users-list', {
title: 'Home Page',
users: responseBody,
message: message
});
}
const homelist = (req, res) => {
const path = '/api/users';
const requestOptions = {
url: `${apiOptions.server}${path}`,
method: 'GET',
json: true,
};
request(
requestOptions,
(err, {statusCode}, body) => {
if (err) {
console.log('Ther was an error ', err);
} else if (statusCode === 200 && body.length) {
renderHomepage(req, res, body);
} else if (statusCode !== 200 && !body.length) {
console.log('error ',statusCode);
}
}
);
}
I've searched extensively on both here and other resources but none of the solutions quite answers my question. I hope someone could be of help

How to wait for a url callback before send HTTP response in koa?

I have a koa router I need to call a api where will async return result. This means I cannot get my result immediately, the api will call my callback url when it's ok. But now I have to use it like a sync api which means I have to wait until the callback url is called.
My router like this:
router.post("/voice", async (ctx, next) => {
// call a API here
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
// data here is not the result I want, this api just return a task id, this api will call my url back
const taskid = data.taskid;
// now I want to wait here until I got "ret_callback"
// .... wait .... wait
// "ret_callback" is called now
// get the answer in "ret_callback"
ctx.body = {
result: "ret_callback result here",
}
})
my callback url like this:
router.post("/ret_callback", async (ctx, next) => {
const params = ctx.request.body;
// taskid will tell me this answer to which question
const taskid = params.taskid;
// this is exactly what I want
const result = params.text;
ctx.body = {
code: 0,
message: "success",
};
})
So how can I make this aync api act like a sync api?
Just pass a resolve() to another function. For example, you can do it this way:
// use a map to save a lot of resolve()
const taskMap = new Map();
router.post("/voice", async (ctx, next) => {
// call a API here
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
const result = await waitForCallback(data.taskid);
ctx.body = {
result,
} })
const waitForCallback = (taskId) => {
return new Promise((resolve, reject) => {
const task = {};
task.id = taskId;
task.onComplete = (data) => {
resolve(data);
};
task.onError = () => {
reject();
};
taskMap.set(task.id, task);
});
};
router.post("/ret_callback", async (ctx, next) => {
const params = ctx.request.body;
// taskid will tell me this answer to which question
const taskid = params.taskid;
// this is exactly what I want
const result = params.text;
// here you continue the waiting response
taskMap.get(taskid).onComplete(result);
// not forget to clean rubbish
taskMap.delete(taskid);
ctx.body = {
code: 0,
message: "success",
}; })
I didn't test it but I think it will work.
function getMovieTitles(substr) {
let movies = [];
let fdata = (page, search, totalPage) => {
let mpath = {
host: "jsonmock.hackerrank.com",
path: "/api/movies/search/?Title=" + search + "&page=" + page,
};
let raw = '';
https.get(mpath, (res) => {
res.on("data", (chunk) => {
raw += chunk;
});
res.on("end", () => {
tdata = JSON.parse(raw);
t = tdata;
totalPage(t);
});
});
}
fdata(1, substr, (t) => {
i = 1;
mdata = [];
for (i = 1; i <= parseInt(t.total_pages); i++) {
fdata(i, substr, (t) => {
t.data.forEach((v, index, arrs) => {
movies.push(v.Title);
if (index === arrs.length - 1) {
movies.sort();
if (parseInt(t.page) === parseInt(t.total_pages)) {
movies.forEach(v => {
console.log(v)
})
}
}
});
});
}
});
}
getMovieTitles("tom")
Okay so first of all, this should not be a "goal" for you. NodeJS works better as ASync.
However, let us assume that you still want it for some reason, so take a look at sync-request package on npm (there is a huge note on there that you should not this in production.
But, I hope you mean on how to make this API simpler (as in one call kinda thingy). You still need .next or await but it will be be one call anyway.
If that is the case, please comment on this answer I can write you a possible method I use myself.
How about this ?
router.post("/voice", async (ctx, next) => {
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
// data here is not the result I want, this api just return a task id, this api will call my url back
const taskid = data.taskid;
let response = null;
try{
response = await new Promise((resolve,reject)=>{
//call your ret_callback and when it finish call resolve(with response) and if it fails, just reject(with error);
});
}catch(err){
//errors
}
// get the answer in "ret_callback"
ctx.body = {
result: "ret_callback result here",
}
});

Resources