DynamoDB, Lambda function / custom module Timeout - node.js

I have the following two JS file. My problem is when i call the Calls.js which calls the Archive.js for archiving logs into DynamoDB the request times out.
I have tried out, many things, read about many things, tried in local/AWS environment without luck. What am i missing?
Link1, Link2, Link3, Link4, Link5,
Archive.js
module.exports.archive = archive;
...
function archive(input, callback){
AWS.config.update({
region: "eu-west-1",
endpoint: "http://localhost:8000"
});
var documentClient = new AWS.DynamoDB.DocumentClient({
httpOptions: {
agent: new https.Agent({
rejectUnauthorized: true,
secureProtocol: "TLSv1_method",
ciphers: "ALL"
})
}
});
...
var paramsPUT = {
TableName: "Logging",
Item: {
HashKey: dbID,
archiveEntry: archiveEntry
}
};
...
documentClient.put(paramsPUT, function(err, data) {
if (err) console.log(err);
if (data) console.log(data);
...
callback(data);
});
}
Calls.js
exports.handler(event, context, callback) => {
const archive = require("./..path..").archive;
...
context.callbackWaitsForEmptyEventLoop = false;
...
archive(input, callback);
...
}

I can not reproduce a timeout condition with your code. Your code is talking to an AWS endpoint at http://localhost:8000, so I assume you have DynamoDB local up and running, don't you ? Failling to have local DynamoDB running would cause the timeout.
That being said, I would strongly suggest to refactor your code to use Promise and the new async/await provided by NodeJS 8 instead of passing the Lambda callback around.
Here is the modified code.
const AWS = require("aws-sdk");
async function archive(input) {
return new Promise( (resolve, reject) => {
AWS.config.update({
region: "eu-west-1",
endpoint: 'http://localhost:8000'
});
//use client specific AWS configuration instead of the global one
const documentClient = new AWS.DynamoDB.DocumentClient();
var paramsPUT = {
TableName: "Logging",
Item: {
HashKey: "123",
archiveEntry: input
}
};
documentClient.put(paramsPUT, function (err, data) {
if (err) {
console.log("ERROR " + err);
reject(err);
}
console.log("Returned from DDB " + JSON.stringify(data, null,2));
resolve(data);
});
});
}
exports.handler = async (event, context, callback) => {
const result = await archive("abc");
callback(result);
}
// stuffs to test locally
callback = function (data) {
console.log("callback called with " + JSON.stringify(data,null,2));
}
event = context = {}
exports.handler(event, context, callback);

Related

Inconsistent DynamoDB writtings

We have the following code used as lambda Function in Serverless Framework triggered every 2min with cron. The issue we are facing is that the writing in DynamoDB is inconsistent , we want to have 3 writings but instead we receive 1 or 2 writings every 2 minutes.
DynamoDB has a HASH key the HOUR and SORT KEY the DATE and Billing mode: PROVISIONED. Has someone faced the same behavior from DynamoDB or the same issue to share how he sovled it. Thanks
"use strict";
const AWS = require("aws-sdk");
const axios = require("axios");
const dynamoDb = new AWS.DynamoDB.DocumentClient();
const lambda = new AWS.Lambda({
region: "us-east-1",
});
module.exports.getWeather = async (event, context, callback) => {
const openWeatherMapAPIURL = `http://api.openweathermap.org/data/2.5/weather?id=${event}&appid=XXXXXXXXXXXXXXXXXXXXXXX&units=metric`;
const currentWeather = await axios
.get(openWeatherMapAPIURL)
.then((records) => {
console.log(records);
const d = new Date(records.headers.date);
let hour = d.getHours();
const params = {
TableName: process.env.DYNAMODB_TABLE_NAME,
Item: {
hour: hour,
date: records.headers.date,
city: records.data.name,
temp: records.data.main.temp,
feelsLike: records.data.main.feels_like,
description: records.data.weather[0].description,
},
};
setTimeout(function () {
dynamoDb.put(params, (error) => {
// handle potential errors
console.log(`zapis na: ${records.data.name} ${records.headers.date}`);
if (error) {
console.log(error);
console.error(error);
return;
}
});
}, 3000);
})
.catch((error) => {
console.log(error);
return;
});
const response = {
statusCode: 200,
body: JSON.stringify({
message: `Weather from ${event} was requested!`,
}),
};
callback(null, response);
};
module.exports.cron_launcher = (event, context, callback) => {
const requestedID = ["786735", "792578", "785842"];
requestedID.forEach((requestedID) => {
const params = {
FunctionName: process.env.HANDLER_LOCATION + "-getWeather",
InvocationType: "RequestResponse",
Payload: JSON.stringify(requestedID),
};
return lambda.invoke(params, function (error, data) {
if (error) {
console.error(JSON.stringify(error));
return new Error(`Error printing messages: ${JSON.stringify(error)}`);
} else if (data) {
console.log(data);
}
});
});
};
You are not waiting for the dynamodb.put operation to finish. Additionally, you are wrapping the call in a setTimeout. Your lambda function is returning before the network operation can be made. Make sure the put operation succeeds before returning a result from your lambda.
I see no reason for you to use a setTimeout here.
You can call dynamodb.put(...).promise() to get a promise from the dynamodb SDK and await that promise.
2.a Or you can continue using a callback, but wrap the entire section of code in a new promise object, calling the resolve method after the dynamodb.put call finishes.

Not able to fetch data from dynamodb from lambda function

I am trying to fetch data from dynamodb from my lambda.I have written this code
exports.handler = async (event) => {
// TODO implement
var AWS = require('aws-sdk');
AWS.config.update({region: 'ap-south-1'});
var ddb = new AWS.DynamoDB({apiVersion: '2012-08-10'});
var params = {
TableName: 'my_table',
Key: {
'serial_number': {S: '17AB-574C-C1'}
},
};
ddb.getItem(params, function(err, data) {
if (err) {
console.log("Error", err);
} else {
console.log("Success", data.Item);
console.log(data);
}
});
};
This code works fine when I run locally but I get a null response when I run it on lambda.My lambda has dynamoDbfullAccess policy attached to it. Can anyone tell me what could be the reason?

Listing cognito userpool users on AWS lambda

I'm trying to list all of my cognito users in my lambda function, however i get nothing in the return as if the callback not getting executed. What am I doing wrong?
The output of the code below just gives me a hello in the console.
var AWS = require("aws-sdk");
const cognitoidentityserviceprovider = new AWS.CognitoIdentityServiceProvider();
export async function main() {
console.log("hello")
var params = {
UserPoolId: "myuserpoolid",
AttributesToGet: ["username"]
};
cognitoidentityserviceprovider.listUsers(params, (err, data) => {
if (err) {
console.log(err, err.stack);
return err;
} else {
console.log(data);
return data;
}
});
}
First of all, the structure of the code is wrong. The header of Lambda function should have a certain structure, either using async function or non-async function. Since you are using non-async code in your example I will show you how to do the later.
var AWS = require("aws-sdk");
const cognitoidentityserviceprovider = new AWS.CognitoIdentityServiceProvider();
exports.handler = function(event, context, callback) {
console.log("hello")
var params = {
UserPoolId: "myuserpoolid",
AttributesToGet: ["username"]
};
cognitoidentityserviceprovider.listUsers(params, (err, data) => {
if (err) {
console.log(err, err.stack);
callback(err) // here is the error return
} else {
console.log(data);
callback(null, data) // here is the success return
}
});
}
In this case, Lambda will finish only when callback is called (or when it times out).
Similarly, you can use async function but you will need to restructure your code accordingly. Here is an example taken from official docs. Note how the promise wrapper is used.
const https = require('https')
let url = "https://docs.aws.amazon.com/lambda/latest/dg/welcome.html"
exports.handler = async function(event) {
const promise = new Promise(function(resolve, reject) {
https.get(url, (res) => {
resolve(res.statusCode)
}).on('error', (e) => {
reject(Error(e))
})
})
return promise
}
For AttributesToGet, don't use username because it is one of the fields that always gets returned. The following are members of the Attributes array, and can be used in the AttributesToGet field:
sub, email_verified, phone_number_verified, phone_number, email.
e.g.
AttributesToGet: ["email","email_verified"]

Cant insert data into DynamoDB using new nodejs 8.10

I want to use new nodejs 8.10 for developing my lambdas. A simple piece of code when written in node 6.10 style works but the same(similar) code doesn't work when I use node 8.10.
Below is working code which successfully inserts data into dynamodb table(nodejs 6.10)
var AWS = require('aws-sdk');
// Set the region
AWS.config.update({region: 'us-east-1'});
var documentClient = new AWS.DynamoDB.DocumentClient({apiVersion: '2012-08-10'});
exports.handler = (event, context, callback) => {
// TODO implement
var params = {
Item: {
client: 'client_'+Math.random(),
Type: 1,
Status: true,
json: { foo: 'bar', address:{ city:'Pune', street: 'ABC Nagar', pin:'411099'} }
},
TableName: 'clients'
};
documentClient.put(params, function(err, data) {
if (err) {
console.log("Error", err);
callback(err, null);
} else {
console.log("Success", data);
// return "Hi, insert data completed";
callback(null, data);
}
});
};
And below one which is node 8.10 style which doesn't work(means doesn't insert data into dynamodb table). I keep getting null as return value.
var AWS = require('aws-sdk');
// Set the region
AWS.config.update({region: 'us-east-1'});
var documentClient = new AWS.DynamoDB.DocumentClient({apiVersion: '2012-08-10'});
exports.handler = async (event) => {
// TODO implement
var params = {
Item: {
client: 'client_'+Math.random(),
Type: 1,
Status: true,
json: { foo: 'bar', address:{ city:'Pune', street: 'ABC Nagar', pin:'411099'} }
},
TableName: 'clients'
};
documentClient.put(params, function(err, data) {
if (err) {
console.log("Error", err);
} else {
console.log("Success", data);
return "Hi, insert data completed";
}
});
};
I spent searching 2-3 hours searching.. couldn't find any article or any question similar. Can anyone tell me what am I doing wrong?
Async / Await is a syntactical sugar for promise, Your documentClient.put should be wraped with promise. Since documentClient.put is based on callback appoach, you have to wrap it with promise
var AWS = require('aws-sdk');
// Set the region
AWS.config.update({region: 'us-east-1'});
var documentClient = new AWS.DynamoDB.DocumentClient({apiVersion: '2012-08-10'});
exports.handler = async (event) => {
// TODO implement
var params = {
Item: {
client: 'client_'+Math.random(),
Type: 1,
Status: true,
json: { foo: 'bar', address:{ city:'Pune', street: 'ABC Nagar', pin:'411099'} }
},
TableName: 'clients'
};
let putItem = new Promise((res, rej) => {
documentClient.put(params, function(err, data) {
if (err) {
console.log("Error", err);
rej(err);
} else {
console.log("Success", data);
res("Hi, insert data completed");
}
});
});
const result = await putItem;
console.log(result);
return result
};
Note: Its advisable to use DB operations in separate file,rather than using in handler function itself
Did you look in your table to see if it's inserting data? I think it is.
The problem with your async-style code is that you aren't returning a value. Returning "Hi, insert data completed" from the put callback doesn't return a value from handler.
You could manually create a promise and return that from handler, but I'd try using promisify.
This code is untested but should be close:
...
const util = require('util');
...
documentClient.putPromise = util.promisify(documentClient.put);
...
try {
const data = await documentClient.putPromise(params);
console.log("Success", data);
return "Hi, insert data completed";
}
catch (err) {
console.log("Error", err);
}
Here's more on promisify: http://2ality.com/2017/05/util-promisify.html
Calling await dynamo.put(params).promise(); is how I solved this issue after some googling. Specifically, it seems like calling foo.promise(); in the aws sdk is supported now.

SQS to Lambda + SES

I'm new with Lambda & SQS and I'm trying to create a function to send emails, queued in an SQS service, but I don't understand how to call the process function that contains the send + delete queue methods.
Here bellow I paste my code:
'use strict';
const AWS = require('aws-sdk');
const SQS = new AWS.SQS({ apiVersion: '2012-11-05' });
const Lambda = new AWS.Lambda({ apiVersion: '2015-03-31' });
const ses = new AWS.SES({ accessKeyId: "xxxxxxxx", secretAccesskey: "xxxxxxx/xxxxxxxxx" });
const s3 = new AWS.S3({ apiVersion: "2006-03-01", region: "us-west-2" });
const QUEUE_URL = 'https://sqs.us-west-2.amazonaws.com/xxxxxxx/queue';
const PROCESS_MESSAGE = 'process-message';
function getPieceOfMail (path, mapObj, replace) {
return new Promise(function (resolve, reject) {
s3.getObject({
Bucket: "myBucket",
Key: "myKey/" + path
}, function (err, data) {
if (err) {
reject(err);
} else {
if (replace === true) {
var re = new RegExp(Object.keys(mapObj).join("|"), "gi");
data = data.Body.toString().replace(re, function (matched) {
return mapObj[matched.toLowerCase()];
});
resolve(data);
} else {
resolve(data.Body.toString());
}
}
});
});
}
function getRegisterSource (nickname, activate_link) {
var activate_link, pieces;
pieces = [
getPieceOfMail("starts/start.html", {}, false),
getPieceOfMail("headers/a.html", {}, false),
getPieceOfMail("footers/a.html", {}, false),
];
return Promise.all(pieces)
.then(function (data) {
return (data[0] + data[1] + data[2]);
})
.catch(function (err) {
return err;
});
}
function sendEmail (email, data) {
return new Promise(function (resolve, reject) {
var params = {
Destination: { ToAddresses: [email] },
Message: {
Body: {
Html: {
Data: data
},
Text: {
Data: data
}
},
Subject: {
Data: "myData"
}
},
Source: "someone <noreply#mydomain.co>",
};
ses.sendEmail(params, function (err, data) {
if (err) {
reject(err);
} else {
resolve(data);
}
});
});
}
function process(message, callback) {
console.log(message);
// process message
getRegisterSource(event['nickname'], event['user_id'])
.then(function (data) {
return sendEmail(event["email"], data);
})
.catch(function (err) {
console.log("==ERROR==");
callback(err, err);
})
.finally(function () {});
// delete message
const params = {
QueueUrl: QUEUE_URL,
ReceiptHandle: message.ReceiptHandle,
};
SQS.deleteMessage(params, (err) => callback(err, message));
}
function invokePoller(functionName, message) {
const payload = {
operation: PROCESS_MESSAGE,
message,
};
const params = {
FunctionName: functionName,
InvocationType: 'Event',
Payload: new Buffer(JSON.stringify(payload)),
};
return new Promise((resolve, reject) => {
Lambda.invoke(params, (err) => (err ? reject(err) : resolve()));
});
}
function poll(functionName, callback) {
const params = {
QueueUrl: QUEUE_URL,
MaxNumberOfMessages: 10,
VisibilityTimeout: 10,
};
// batch request messages
SQS.receiveMessage(params, (err, data) => {
if (err) {
return callback(err);
}
// for each message, reinvoke the function
const promises = data.Messages.map((message) => invokePoller(functionName, message));
// complete when all invocations have been made
Promise.all(promises).then(() => {
const result = `Messages received: ${data.Messages.length}`;
callback(null, result);
});
});
}
exports.handler = (event, context, callback) => {
try {
if (event.operation === PROCESS_MESSAGE) {
console.log("Invoked by poller");
process(event.message, callback);
} else {
console.log("invoked by schedule");
poll(context.functionName, callback);
}
} catch (err) {
callback(err);
}
};
can somebody throw me some light to this?
Thanks in advice.
UPDATE
After so much misconception, I've decided to start looking on how the example of polling-SQS works provided by AWS.
There I've found that I lacked some basic SQS permissions, but solved now by adding the right policy:
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": [
"lambda:InvokeFunction"
],
"Resource": ["*"]
}]
}
This allows Lambda.invoke() to call process().
When the process(message, callback) is called, if I console.log(message);, it seems that there's no message, although the queue is being cleared by the line SQS.deleteMessage(params, (err) => callback(err, message));
What I was trying was to combine my sendMail function that is currently working with a SQS service so I only have to push each message to the queue.
This is a common requirement where AWS SES has its own limitations in sending emails at once. If these limitations are violated, the SES account will sandbox itself. It seems like you have solved the problem using proper access credentials.
This code contains a Python3 Lambda code that can be used to handle a situation like this, where a Lambda polls from SQS using threading, and sends emails using SES, without exceeding the given limitations.
Link to Github Project.
You can also consider using the new feature in SQS, which is capable of invoking lambdas, when a new message is placed within SQS. But, be careful not to exceed the maximum number of lambda functions within the AWS Account region. (See this document)

Resources