Why does AWS Lambda function always time out? - node.js

I am testing out aws lambda, using nodejs with the 4.3 version. I'm able to successfully complete all the statements in my handler function within the console test, which includes connecting to a mongodb host within our vpc. But, the function always times out. I've found several posts and resources that discuss using the callback, and setting properties on the context, and IAM role permissions, but no matter what I do, it always ends up timing out. Current code:
'use strict';
var Mongoose = require('mongoose');
var Device = require('./device_model');
var Alarm = require('./alarm_model');
var Event = require('./event_model');
var mongoConnection = process.env.MONGO_URL;
var connection = Mongoose.connect(mongoConnection);
Mongoose.connection.once('open', function() {
console.log("Connecting to mongo at: " + mongoConnection);
console.log("Mongoose connection in lambda opened");
});
Mongoose.connection.on('error', function(){
console.error("Error creating mongoose connection in lambda, exiting!");
process.exit(1);
});
exports.check_alarms = function(event, context, callback) {
context.callbackWaitsForEmtpyEventLoop = false;
console.log("The incoming event: " + JSON.stringify(event));
var device = null;
Device.findByUUID(event.uuid, function(error, result){
if(!error){
device = result;
console.log("the device: " + JSON.stringify(device));
if(event.Ale && event.Ale.length > 0) {
console.log("We have an alarm, checking if already set");
callback(null, {"status":"alarms"});
} else {
console.log("Event contains no alarm; checking for historic active");
callback(null, {"status":"no alarms"});
}
} else {
console.log("there's a problem on mongo");
callback("problem", "status not so good");
}
});
callback(null, {"status":"outside of device find block"});
}

You have a typo:
context.callbackWaitsForEmtpyEventLoop = false;
should be:
context.callbackWaitsForEmptyEventLoop = false;
Here's what the documentation says about the behavior of callbackWaitsForEmptyEventLoop:
callbackWaitsForEmptyEventLoop
The default value is true. This property is useful only to modify the default behavior of the callback. By default, the callback will wait until the Node.js runtime event loop is empty before freezing the process and returning the results to the caller. You can set this property to false to request AWS Lambda to freeze the process soon after the callback is called, even if there are events in the event loop. AWS Lambda will freeze the process, any state data and the events in the Node.js event loop (any remaining events in the event loop processed when the Lambda function is called next and if AWS Lambda chooses to use the frozen process). For more information about callback, see Using the Callback Parameter.
Minimal example:
// Times out due to typo
exports.function1 = (event, context, callback) => {
setInterval(() => console.log('Long wait'), 100000);
context.callbackWaitsForEmtpyEventLoop = false;
callback(null, 'Hello from Lambda');
};
// Returns successfully
exports.function2 = (event, context, callback) => {
setInterval(() => console.log('Long wait'), 100000);
context.callbackWaitsForEmptyEventLoop = false;
callback(null, 'Hello from Lambda');
};

If anybody was confused like I was with how to add callbackWaitsForEmptyEventLoop to new Alexa projects that look like this:
const skillBuilder = Alexa.SkillBuilders.custom();
exports.handler = skillBuilder
.addRequestHandlers(
GetNewFactHandler,
HelpHandler,
ExitHandler,
FallbackHandler,
SessionEndedRequestHandler,
)
.addRequestInterceptors(LocalizationInterceptor)
.addErrorHandlers(ErrorHandler)
.lambda();
Example project found here: https://github.com/alexa/skill-sample-nodejs-fact/blob/master/lambda/custom/index.js
This solution worked for me:
// create a custom skill builder
const skillBuilder = Alexa.SkillBuilders.custom();
exports.handler = (event, context, callback) => {
// we need this so that async stuff will work better
context.callbackWaitsForEmptyEventLoop = false
// set up the skill with the new context
return skillBuilder
.addRequestHandlers(
GetNewFactHandler,
HelpHandler,
ExitHandler,
FallbackHandler,
SessionEndedRequestHandler,
)
.addRequestInterceptors(LocalizationInterceptor)
.addErrorHandlers(ErrorHandler)
.lambda()(event, context, callback);
}

I was triggering a lambda from android code. The lambda was performing a heavy data lifting operation from DynamoDB and generating a JS object as a response. I could see the generated object in my lambda logs, but my android side code always timed out waiting for a response.
The reason for timeout was the Invocation payload (request and response) limit of 6 MB for synchronous and 256 KB for asynchronous lambda implementation. In my case I had a asynchronous implementation and the resultant object was over 256 KB.
There was no appropriate log message for overshooting this limit which derailed me. Reference: https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-limits.html
The workaround I implemented was writing the response to a text file in S3 bucket and passing its reference as a response to the android side instead of passing the actual object itself.

Related

Lambda function not pushing data into DynamoDB table

I'm running a NodeJS lambda function which is triggered by API Gateway.
My goal is to push the data and then send a status response. I think the lambda stops running before the insertData function finishes its execution, because sometimes it works but in most requests it doesn't.
Could someone lend a hand on this?
Here is my code:
// Set a table name that we can use later on
const tableName = "InterestRates"
// Load the AWS SDK for Node.js
var AWS = require('aws-sdk');
// Set the region
AWS.config.update({region: 'us-east-1'});
// Create the DynamoDB service object
var ddb = new AWS.DynamoDB({apiVersion: '2012-08-10'});
exports.handler = async (event) => {
// TODO implement
console.log(">>> Running LoanInterestRates")
if(event['resource'] == '/rates'){
if(event.httpMethod === 'POST'){
return newRate(event);
}
}
};
function insertData(params){
let status;
// Call DynamoDB to add the item to the table
ddb.putItem(params, function(err, data) {
if (err) {
status = "Error";
} else {
status = "Success";
}
});
return status
}
function newRate (event){
const newRate = JSON.parse(event.body);
var params = {
TableName: 'InterestRates',
Item: {
'termlength' : {N: newRate["length"]},
'InterestRate': {S: newRate["rate"]}
}
};
let addNewRate = insertData(params);
return {
statusCode: 200,
body: JSON.stringify({
response: addNewRate
})
}
}
I also tried using Async/Await but it didn't work.
You lambda function is Async but your code is not. You need to await the completion of your function newRate which in turn should also await the function inserData which should also await your DDB request.
I would advise you to do one of two things:
Learn how JS Async nature works and ensure you understand when you need to await.
Use a synchronous programming language like Python/Boto3 where you will not run into such issues.

AWS SQS Node.js script does not await

Trying to send several messages (from AWS SQS lambda, if that matters) but it's never waiting for the promises.
function getEndpoint(settings){
return new Promise(function(resolve, reject) {
// [...] more stuff here
}
Which is then called in a loop:
exports.handler = async (event) => {
var messages = [];
event.Records.forEach(function(messageId, body) {
//options object created from some stuff
messages.push(getEndpoint(options).then(function(response){
console.log("anything at all"); //NEVER LOGGED
}));
});
await Promise.all(messages);
};
But the await seems to be flat out skipped. I'm not sure how I'm getting Process exited before completing request with an explicit await. I have similar async await/promise setups in other scripts that work, but cannot spot what I've done wrong with this one.
You forgot to return something to lambda:
exports.handler = async (event) => {
var messages = [];
event.Records.forEach(function(messageId, body) {
//options object created from some stuff
messages.push(getEndpoint(options));
});
await Promise.all(messages);
return 'OK'
};
this should also work:
exports.handler = (event) => { // async is not mandatory here
var messages = [];
event.Records.forEach(function(messageId, body) {
//options object created from some stuff
messages.push(getEndpoint(options));
});
return Promise.all(messages); // returning a promise
};
and you could use map:
exports.handler = (event) => { // async is not mandatory here
const messages = event.Records.map(function(messageId, body) {
//options object created from some stuff
return getEndpoint(options)
});
return Promise.all(messages); // returning a promise
};
To understand why this happens, you must dive a bit into lambda's implementation: it will essentially wait for the function stack to be cleared and since you did NOT return anything at all in there, the function stack got empty right after it queued all the stuff - adding a simple return after the await call makes the fn stack to NOT be empty which means lambda will wait for it to be finished.
If you run this on standard node, your function would also return before the promises were finished BUT your node process would NOT exit until the stack was cleared. This is where lambda diverges from stock node.

AWS Lambda function flow

I'm having some issues with how my functions flow in lambda. I'm trying to grab value stored in S3, increment it, and put it back. However, my program doesn't flow how I feel it should be. I'm using async waterfall to run the flow of my functions.
Here's my code:
let AWS = require('aws-sdk');
let async = require('async');
let bucket = "MY_BUCKET";
let key = "MY_FILE.txt";
exports.handler = async (event) => {
let s3 = new AWS.S3();
async.waterfall([
download,
increment,
upload
], function (err) {
if (err) {
console.error(err);
} else {
console.log("Increment successful");
}
console.log("test4");
return null;
}
);
console.log("test5");
function download(next) {
console.log("test");
s3.getObject({
Bucket: bucket,
Key: key
},
next);
}
function increment(response, next) {
console.log("test2");
console.log(response.Body);
let newID = parseInt(response.Body, 10) + 1;
next(response.ContentType, newID);
}
function upload(contentType, data, next) {
console.log("test3");
s3.putObject({
Bucket: bucket,
Key: key,
Body: data,
ContentType: contentType
},
next);
}
};
I'm only getting test and test5 on my log. I was under the impression that after the download function, increment should run if it was okay or the callback function at the end of the waterfall should run if there was an error. The program doesn't give an error on execution and it doesn't appear to go to either function.
Could someone guide me to what I'm missing in my understanding?
EDIT: So it seems my issue was related to my function declaration. The default template declared it as async(event). I thought this was odd as usually they are declared as (event, context, callback). Switching to the later (or even just (event) without the async) fixed this. It looks like my issue is with calling the function as asynchronous. This blocked the waterfall async calls?? Can anyone elaborate on this?
Your problem is that your handler is declared as an async function, which will create a promise for you automatically, but since you are not awaiting at all your function is essentially ending synchronously.
There are a couple of ways to solve this, all of which we'll go over.
Do not use promises, use callbacks as the async library is designed to use.
Do not use the async library or callbacks and instead use async/await.
Mix both together and make your own promise and resolve/reject it manually.
1. Do not use promises
In this solution, you would remove the async keyword and add the callback parameter lambda is passing to you. Simply calling it will end the lambda, passing it an error will signal that the function failed.
// Include the callback parameter ────┐
exports.handler = (event, context, callback) => {
const params =[
download,
increment,
upload
]
async.waterfall(params, (err) => {
// To end the lambda call the callback here ──────┐
if (err) return callback(err); // error case ──┤
callback({ ok: true }); // success case ──┘
});
};
2. Use async/await
The idea here is to not use callback style but to instead use the Promise based async/await keywords. If you return a promise lambda will use that promise to handle lambda completion instead of the callback.
If you have a function with the async keyword it will automatically return a promise that is transparent to your code.
To do this we need to modify your code to no longer use the async library and to make your other functions async as well.
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
const Bucket = "MY_BUCKET";
const Key = "MY_FILE.txt";
async function download() {
const params = {
Bucket,
Key
}
return s3.getObject(params).promise(); // You can await or return a promise
}
function increment(response) {
// This function is synchronous, no need for promises or callbacks
const { ContentType: contentType, Body } = response;
const newId = parseInt(Body, 10) + 1;
return { contentType, newId };
}
async function upload({ contentType: ContentType, newId: Body }) {
const params = {
Bucket,
Key,
Body,
ContentType
};
return s3.putObject(params).promise();
}
exports.handler = async (event) => {
const obj = await download(); // await the promise completion
const data = increment(obj); // call synchronously without await
await upload(data)
// The handlers promise will be resolved after the above are
// all completed, the return result will be the lambdas return value.
return { ok: true };
};
3. Mix promises and callbacks
In this approach we are still using the async library which is callback based but our outer function is promised based. This is fine but in this scenario we need to make our own promise manually and resolve or reject it in the waterfall handler.
exports.handler = async (event) => {
// In an async function you can either use one or more `await`'s or
// return a promise, or both.
return new Promise((resolve, reject) => {
const steps = [
download,
increment,
upload
];
async.waterfall(steps, function (err) {
// Instead of a callback we are calling resolve or reject
// given to us by the promise we are running in.
if (err) return reject(err);
resolve({ ok: true });
});
});
};
Misc
In addition to the main problem of callbacks vs. promises you are encountering you have a few minor issues I noticed:
Misc 1
You should be using const rather than let most of the time. The only time you should use let is if you intend to reassign the variable, and most of the time you shouldn't do that. I would challenge you with ways to write code that never requires let, it will help improve your code in general.
Misc 2
You have an issue in one of your waterfall steps where you are returning response.ContentType as the first argument to next, this is a bug because it will interpret that as an error. The signature for the callback is next(err, result) so you should be doing this in your increment and upload functions:
function increment(response, next) {
const { ContentType: contentType, Body: body } = response;
const newId = parseInt(body, 10) + 1;
next(null, { contentType, newId }); // pass null for err
}
function upload(result, next) {
const { contentType, newId } = result;
s3.putObject({
Bucket: bucket,
Key: key,
Body: newId,
ContentType: contentType
},
next);
}
If you don't pass null or undefined for err when calling next async will interpret that as an error and will skip the rest of the waterfall and go right to the completion handler passing in that error.
Misc 3
What you need to know about context.callbackWaitsForEmptyEventLoop is that even if you complete the function correctly, in one of the ways discussed above your lambda may still hang open and eventually timeout rather than successfully complete. Based on your code sample here you won't need to worry about that probably but the reason why this can happen is if you happen to have something that isn't closed properly such as a persistent connection to a database or websocket or something like that. Setting this flag to false at the beginning of your lambda execution will cause the process to exit regardless of anything keeping the event loop alive, and will force them to close ungracefully.
In the case below your lambda can do the work successfully and even return a success result but it will hang open until it timesout and be reported as an error. It can even be re-invoked over and over depending on how it's triggered.
exports.handler = async (event) => {
const db = await connect()
await db.write(data)
// await db.close() // Whoops forgot to close my connection!
return { ok: true }
}
In that case simply calling db.close() would solve the issue but sometimes its not obvious what is hanging around in the event loop and you just need a sledge hammer type solution to close the lambda, which is what context.callbackWaitsForEmptyEventLoop = false is for!
exports.handler = async (event) => {
context.callbackWaitsForEmptyEventLoop = false
const db = await connect()
await db.write(data)
return { ok: true }
}
The above will complete the lambda as soon as the function returns, killing all connections or anything else living in the event loop still.
Your function terminates before the waterfall is resolved. That is, the asynchronous calls aren't executed at all. That is why you don't see any of the console.log calls you have within the waterfall functions, and only see the one that is called synchronously immediately after the call to async.waterfall.
Not sure how well async.waterfall is supported by AWS Lambda, but since promises are natively supported and perform the same functionality (with fewer loc), you could use promises instead. Your code would look something like this:
module.exports.handler = (event,context) =>
s3.getObject({
Bucket: bucket,
Key: key
}).promise()
.then(response => ({
Body: parseInt(response.Body, 10) + 1,
ContentType: response.contentType,
}))
.then(modifiedResponse => s3.putObject({
Bucket: bucket,
Key: key,
Body: modifiedResponse.data,
ContentType: modifiedResponse.contentType}).promise())
.catch(err => console.error(err));

Why my lambda is working only when i give a callback with some message?

I am trying to run a small snippet of lambda code where i am pushing data to S3 using firehose. Here is my snippet
const AWS = require( 'aws-sdk' );
var FIREhose = new AWS.Firehose();
exports.handler = async (event,context,callback) => {
// TODO implement
const response = {
statusCode:200,
Name:event.Name,
Value:event.Value
};
const params = {
DeliveryStreamName: 'kinesis-firehose',
Record: { Data: new Buffer(JSON.stringify(response)) }
};
FIREhose.putRecord(params, (err, data) => {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data);
});
};
Here are my events
{
"Name": "Mike",
"Value": "66"
}
When i run this lambda all i am getting response as null . Since i am not passing any callback lambda will default run the implicit callback and returns null. I see that no data is pushed to S3 bucket.
But when i add callback(null,"success") line at the end like this
FIREhose.putRecord(params, (err, data) => {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data);
});
callback(null,"success")
};
I see the data is pushed to S3. Why is that ?
Does async functions always need a callback with some text appended to it ?
Any help is appreciated ?
Thanks
The problem here is that you're mixing your node.js lambda patterns.
Either you use an asynchronous function and return or throw:
exports.handler = async (event,context,callback) => {
// code goes here.
await FIREhose.putRecord(params).promise();
return null; // or whatever result.
};
Or you use the callback approach:
exports.handler = (event,context,callback) => {
// code goes here.
FIREhose.putRecord(params)
.promise();
.then((data) => {
// do stuff with data.
// n.b. you could have used the cb instead of a promise here too.
callback(null, null); // or whatever result.
});
};
(There's a third way using context. but that's a very legacy way).
This is all due to how lambda works and detects when there's been a response.
In your first example (no callback), lambda is expecting your handler to return a promise that it has to wait to resolve/reject, which, in turn, will be the response. However, you're not returning a promise (undefined) and so there's nothing to wait for and it immediately returns- quite probably before the putRecord call has completed.
When you used callback though, you explicitly told lambda that you're using the "old" way. And the interesting thing about the callback approach is that it waits for node's event loop to complete (by default). Which means that .putRecord will probably complete.

Sharing DB Connection across AWS Lambda function calls

So I'm following the example here https://www.mongodb.com/blog/post/optimizing-aws-lambda-performance-with-mongodb-atlas-and-nodejs, to optimize my lambda functions.
I've tried two approaches and tested them locally using serverless-offline and both don't seem to work.
First Approach
// endpoint file
import {connectToDatabase} from "lib/dbUtils.js";
let cachedDb = null;
export function post(event, context, callback) {
let response;
context.callbackWaitsForEmptyEventLoop = false;
connectToDatabase()
.then(//do other stuff
// lib/dbUtils.js
export async function connectToDatabase() {
if (cachedDb && cachedDb.serverConfig.isConnected()) {
console.log(" using cached db instance");
return cachedDb;
}
cachedDb = await mongoose.createConnection(
process.env.DB_URL,
async err => {
if (err) {
throw err;
}
}
);
return cachedDb;
}
Second Approach
global.cachedDb = null;
export function post(event, context, callback) {
let response;
context.callbackWaitsForEmptyEventLoop = false;
connectToDatabase()
.then(connection => createUser(event.body, connection))
// lib/dbUtils.js
export async function connectToDatabase() {
// eslint-disable-next-line
if (global.cachedDb && global.cachedDb.serverConfig.isConnected()) {
// eslint-disable-next-line
console.log(" using cached db instance");
// eslint-disable-next-line
return global.cachedDb;
}
// eslint-disable-next-line
global.cachedDb = await mongoose.createConnection(
process.env.DB_URL,
async err => {
if (err) {
throw err;
}
}
);
// eslint-disable-next-line
return global.cachedDb;
}
In both cases the using cached db instance console log does not run.
Why does this not work? Is this because of serverless-offline?
The answer is simple: serverless-offline doesn't simulate the full AWS. Use the AWS console to to make a real Lambda
The MongoDB Atlas guide is OK, but it's also worth checking the official AWS Lambda documentation describing the context option in each lambda:
callbackWaitsForEmptyEventLoop – Set to false to send the response right away when the callback executes, instead of waiting for the Node.js event loop to be empty. If false, any outstanding events will continue to run during the next invocation.
It's possible to run your code on a real Lambda and see using cached db instance on the console. Since MongoDB's JavaScript code is fairly poor, I've written out my own version below:
var MongoClient = require("mongodb").MongoClient
let db = null
var log = console.log.bind(console)
var print = function(object) {
return JSON.stringify(object, null, 2)
}
// Use your own credentials (and better yet, put them in environment variables)
const password = `notactuallyapassword`
const uri = `mongodb+srv://lambdauser:${password}#fakedomain.mongodb.net/test?retryWrites=true`
exports.handler = function(event, context, callback) {
log(`Calling MongoDB Atlas from AWS Lambda with event: ${print(event)}`)
var document = JSON.parse(JSON.stringify(event))
const databaseName = "myDatabase",
collectionName = "documents"
// See https://www.mongodb.com/blog/post/optimizing-aws-lambda-performance-with-mongodb-atlas-and-nodejs
// and https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-context.html#nodejs-prog-model-context-properties
context.callbackWaitsForEmptyEventLoop = false
return createDoc(databaseName, collectionName, document)
}
async function createDoc(databaseName, collectionName, document) {
var isConnected = db && db.serverConfig.isConnected()
if (isConnected) {
log(`Already connected to database, warm start!`)
} else {
log(`Connecting to database (cold start)`)
var client = await MongoClient.connect(uri)
db = client.db(databaseName)
}
var result = await db.collection(collectionName).insertOne(document)
log(`just created an entry into the ${collectionName} collection with id: ${result.insertedId}`)
// Don't close the connection thanks to context.callbackWaitsForEmptyEventLoop = false - this will re-use the connection on the next called (if it can re-use the same Lambda container)
return result
}
Use the Test button to run the lambda above twice in the AWS Lambda console.
The first time you run it you'll see Connecting to database (cold start)
The second time you'll see Already connected to database, warm start!
See the log output section in screenshot below:

Resources