Sendgrid on FB Cloud functions, random requests fail with timeout error - node.js

Question

This issue has been happening for a few weeks, some of our emails are not going out, and the SendGrid SDK (#sendgrid/mail for NodeJs) errors out with timeout, we have bumped the timeout time to 20000ms, but we still have some failures.
The way our scheduler system works we save a document in a document DB that holds the metadata for the email and that holds the ID of a GCP task, the GCP task calls happen 100%, and depending on the result it marks the document in the DB as "complete" or "failed", so we notice that those tasks that should be sending an email are marked as failed, and checking our Sentry logs those always have a "timeout" error.
The intriguing part is that some of those errored tasks did actually send an email, so we need to be safe on how to retry those on failure to avoid cases where we send the same email multiple times.
A few key points:
If we don't set a "timeout" value in the Sendgrid config, it won't error out, so the task will be marked as complete even if no email goes out.
Over 95% of our email tasks go out correctly, we were not able to link it to a time of the day or type of the email payload or specific email address, everything looks random, but I am sure there is a reason for those failures
Could it be related to it running in a cloud function? The stack runs on Firebase cloud functions, we have a few debug logs to help it and all the logs show up correctly and the function finalizes the execution correctly
Example of our sender method:
async function sendMail(templateName, msg, worker = {}) {
try {
const API_KEY = functions.config().sendgrid.key;
sgMail.setApiKey(API_KEY);
// https://github.com/sendgrid/sendgrid-nodejs/tree/main/docs/use-cases
sgMail.setTimeout(20000);
const env = getEnvironment();
if (env === "development") {
msg.to = "our-test-email#gmail.com";
}
msg.templateId = functions.config().sendgrid[templateName];
const response = await sgMail.send(msg);
console.log(`DEBUG: ${msg.to} Sendgrid response:`, response);
return response;
} catch (error) {
if (error.response && error.response.body) {
console.error(error.response.body);
} else {
console.log(error);
}
Sentry.withScope(function (scope) {
!isEmpty(worker) &&
scope.setUser({
firstName: user.firstName,
lastName: user.lastName,
});
Sentry.captureException(error);
});
// Throw back the error to the caller to mark the task as failed
throw new functions.https.HttpsError(
"internal",
msg.to ? `Error sending email to ${msg.to}` : "Error sending email",
error
);
}
}
One example of the error:
Error: timeout of 20000ms exceeded
File "/layers/google.nodejs.yarn/yarn_modules/node_modules/#sendgrid/client/node_modules/axios/lib/core/createError.js", line 16, col 15, in createError
var error = new Error(message);
File "/layers/google.nodejs.yarn/yarn_modules/node_modules/#sendgrid/client/node_modules/axios/lib/adapters/http.js", line 303, col 16, in RedirectableRequest.handleRequestTimeout
reject(createError(
File "node:events", line 527, col 28, in RedirectableRequest.emit
File "node:domain", line 537, col 15, in RedirectableRequest.emit
File "/layers/google.nodejs.yarn/yarn_modules/node_modules/follow-redirects/index.js", line 164, col 12, in Timeout.<anonymous>
self.emit("timeout");
File "node:internal/timers", line 559, col 17, in listOnTimeout
File "node:internal/timers", line 502, col 7, in processTimers
Extra response metadata:
scheduler/sender -> sendCompanyEmail() Company: FgTgemCNhhAscqTTMTWa Message: EMAIL HttpsError: Error sending email to ops-dev#trabapro.com
at sendMail (/workspace/lib/src/helpers/index.js:67:15)
at processTicksAndRejections (node:internal/process/task_queues:96:5)
at async /workspace/lib/src/scheduler/sender.js:306:17
at async Promise.all (index 0)
at async sendCompanyEmail (/workspace/lib/src/scheduler/sender.js:303:13) {
code: 'internal',
details: Error: timeout of 20000ms exceeded
at createError (/layers/google.nodejs.yarn/yarn_modules/node_modules/#sendgrid/client/node_modules/axios/lib/core/createError.js:16:15)
at RedirectableRequest.handleRequestTimeout (/layers/google.nodejs.yarn/yarn_modules/node_modules/#sendgrid/client/node_modules/axios/lib/adapters/http.js:303:16)
at RedirectableRequest.emit (node:events:527:28)
at RedirectableRequest.emit (node:domain:537:15)
at Timeout.<anonymous> (/layers/google.nodejs.yarn/yarn_modules/node_modules/follow-redirects/index.js:164:12)
at listOnTimeout (node:internal/timers:559:17)
at processTimers (node:internal/timers:502:7) {
config: {
url: '/v3/mail/send',
method: 'post',
data: ...
...
baseURL: 'https://api.sendgrid.com/',
transformRequest: [Array],
transformResponse: [Array],
timeout: 20000,
adapter: [Function: httpAdapter],
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: Infinity,
maxBodyLength: Infinity,
validateStatus: [Function: validateStatus],
transitional: [Object]
},
code: 'ECONNABORTED',
request: Writable {
_writableState: [WritableState],
_events: [Object: null prototype],
_eventsCount: 3,
_maxListeners: undefined,
_options: [Object],
_ended: true,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 1557,
_requestBodyBuffers: [Array],
_onNativeResponse: [Function (anonymous)],
_currentRequest: [ClientRequest],
_currentUrl: 'https://api.sendgrid.com/v3/mail/send',
_timeout: null,
[Symbol(kCapture)]: false
},
response: undefined,
isAxiosError: true,
toJSON: [Function: toJSON]
},
httpErrorCode: { canonicalName: 'INTERNAL', status: 500 }
```