Cant insert data into DynamoDB using new nodejs 8.10 - node.js

I want to use new nodejs 8.10 for developing my lambdas. A simple piece of code when written in node 6.10 style works but the same(similar) code doesn't work when I use node 8.10.
Below is working code which successfully inserts data into dynamodb table(nodejs 6.10)
var AWS = require('aws-sdk');
// Set the region
AWS.config.update({region: 'us-east-1'});
var documentClient = new AWS.DynamoDB.DocumentClient({apiVersion: '2012-08-10'});
exports.handler = (event, context, callback) => {
// TODO implement
var params = {
Item: {
client: 'client_'+Math.random(),
Type: 1,
Status: true,
json: { foo: 'bar', address:{ city:'Pune', street: 'ABC Nagar', pin:'411099'} }
},
TableName: 'clients'
};
documentClient.put(params, function(err, data) {
if (err) {
console.log("Error", err);
callback(err, null);
} else {
console.log("Success", data);
// return "Hi, insert data completed";
callback(null, data);
}
});
};
And below one which is node 8.10 style which doesn't work(means doesn't insert data into dynamodb table). I keep getting null as return value.
var AWS = require('aws-sdk');
// Set the region
AWS.config.update({region: 'us-east-1'});
var documentClient = new AWS.DynamoDB.DocumentClient({apiVersion: '2012-08-10'});
exports.handler = async (event) => {
// TODO implement
var params = {
Item: {
client: 'client_'+Math.random(),
Type: 1,
Status: true,
json: { foo: 'bar', address:{ city:'Pune', street: 'ABC Nagar', pin:'411099'} }
},
TableName: 'clients'
};
documentClient.put(params, function(err, data) {
if (err) {
console.log("Error", err);
} else {
console.log("Success", data);
return "Hi, insert data completed";
}
});
};
I spent searching 2-3 hours searching.. couldn't find any article or any question similar. Can anyone tell me what am I doing wrong?

Async / Await is a syntactical sugar for promise, Your documentClient.put should be wraped with promise. Since documentClient.put is based on callback appoach, you have to wrap it with promise
var AWS = require('aws-sdk');
// Set the region
AWS.config.update({region: 'us-east-1'});
var documentClient = new AWS.DynamoDB.DocumentClient({apiVersion: '2012-08-10'});
exports.handler = async (event) => {
// TODO implement
var params = {
Item: {
client: 'client_'+Math.random(),
Type: 1,
Status: true,
json: { foo: 'bar', address:{ city:'Pune', street: 'ABC Nagar', pin:'411099'} }
},
TableName: 'clients'
};
let putItem = new Promise((res, rej) => {
documentClient.put(params, function(err, data) {
if (err) {
console.log("Error", err);
rej(err);
} else {
console.log("Success", data);
res("Hi, insert data completed");
}
});
});
const result = await putItem;
console.log(result);
return result
};
Note: Its advisable to use DB operations in separate file,rather than using in handler function itself

Did you look in your table to see if it's inserting data? I think it is.
The problem with your async-style code is that you aren't returning a value. Returning "Hi, insert data completed" from the put callback doesn't return a value from handler.
You could manually create a promise and return that from handler, but I'd try using promisify.
This code is untested but should be close:
...
const util = require('util');
...
documentClient.putPromise = util.promisify(documentClient.put);
...
try {
const data = await documentClient.putPromise(params);
console.log("Success", data);
return "Hi, insert data completed";
}
catch (err) {
console.log("Error", err);
}
Here's more on promisify: http://2ality.com/2017/05/util-promisify.html

Calling await dynamo.put(params).promise(); is how I solved this issue after some googling. Specifically, it seems like calling foo.promise(); in the aws sdk is supported now.

Related

Inconsistent DynamoDB writtings

We have the following code used as lambda Function in Serverless Framework triggered every 2min with cron. The issue we are facing is that the writing in DynamoDB is inconsistent , we want to have 3 writings but instead we receive 1 or 2 writings every 2 minutes.
DynamoDB has a HASH key the HOUR and SORT KEY the DATE and Billing mode: PROVISIONED. Has someone faced the same behavior from DynamoDB or the same issue to share how he sovled it. Thanks
"use strict";
const AWS = require("aws-sdk");
const axios = require("axios");
const dynamoDb = new AWS.DynamoDB.DocumentClient();
const lambda = new AWS.Lambda({
region: "us-east-1",
});
module.exports.getWeather = async (event, context, callback) => {
const openWeatherMapAPIURL = `http://api.openweathermap.org/data/2.5/weather?id=${event}&appid=XXXXXXXXXXXXXXXXXXXXXXX&units=metric`;
const currentWeather = await axios
.get(openWeatherMapAPIURL)
.then((records) => {
console.log(records);
const d = new Date(records.headers.date);
let hour = d.getHours();
const params = {
TableName: process.env.DYNAMODB_TABLE_NAME,
Item: {
hour: hour,
date: records.headers.date,
city: records.data.name,
temp: records.data.main.temp,
feelsLike: records.data.main.feels_like,
description: records.data.weather[0].description,
},
};
setTimeout(function () {
dynamoDb.put(params, (error) => {
// handle potential errors
console.log(`zapis na: ${records.data.name} ${records.headers.date}`);
if (error) {
console.log(error);
console.error(error);
return;
}
});
}, 3000);
})
.catch((error) => {
console.log(error);
return;
});
const response = {
statusCode: 200,
body: JSON.stringify({
message: `Weather from ${event} was requested!`,
}),
};
callback(null, response);
};
module.exports.cron_launcher = (event, context, callback) => {
const requestedID = ["786735", "792578", "785842"];
requestedID.forEach((requestedID) => {
const params = {
FunctionName: process.env.HANDLER_LOCATION + "-getWeather",
InvocationType: "RequestResponse",
Payload: JSON.stringify(requestedID),
};
return lambda.invoke(params, function (error, data) {
if (error) {
console.error(JSON.stringify(error));
return new Error(`Error printing messages: ${JSON.stringify(error)}`);
} else if (data) {
console.log(data);
}
});
});
};
You are not waiting for the dynamodb.put operation to finish. Additionally, you are wrapping the call in a setTimeout. Your lambda function is returning before the network operation can be made. Make sure the put operation succeeds before returning a result from your lambda.
I see no reason for you to use a setTimeout here.
You can call dynamodb.put(...).promise() to get a promise from the dynamodb SDK and await that promise.
2.a Or you can continue using a callback, but wrap the entire section of code in a new promise object, calling the resolve method after the dynamodb.put call finishes.

Not able to fetch data from dynamodb from lambda function

I am trying to fetch data from dynamodb from my lambda.I have written this code
exports.handler = async (event) => {
// TODO implement
var AWS = require('aws-sdk');
AWS.config.update({region: 'ap-south-1'});
var ddb = new AWS.DynamoDB({apiVersion: '2012-08-10'});
var params = {
TableName: 'my_table',
Key: {
'serial_number': {S: '17AB-574C-C1'}
},
};
ddb.getItem(params, function(err, data) {
if (err) {
console.log("Error", err);
} else {
console.log("Success", data.Item);
console.log(data);
}
});
};
This code works fine when I run locally but I get a null response when I run it on lambda.My lambda has dynamoDbfullAccess policy attached to it. Can anyone tell me what could be the reason?

Nodejs 10: Why does DynamoDB put function giving success multiple times

I am writing node js 10.x lambda function to put details into DynamoDB table.
Below is code
const AWS = require('aws-sdk');
var db = new AWS.DynamoDB.DocumentClient();
var tableName="xyz";
exports.handler = async (event) => {
// TODO implement
console.log("Event: "+ JSON.stringify(event));
var response = {
statusCode: 200,
"headers": {
"Access-Control-Allow-Origin" : "*",
"Access-Control-Allow-Credentials" : true
},
};
await db.put({
TableName: tableName,
Item: {
userid: event.userid,
}
}, (error, data) => {
if (error) {
console.log("error:"+ error);
}
else{
console.log("Success");
}
}).promise();
return response;
};
I am getting kind on random number of success return
Output execution 1
2019-11-07T07:03:45.388Z f451dfc1-01ea-41d0-a998-945cb0f18be1 INFO Success
2019-11-07T07:03:45.510Z f451dfc1-01ea-41d0-a998-945cb0f18be1 INFO Success
2019-11-07T07:03:45.511Z f451dfc1-01ea-41d0-a998-945cb0f18be1 INFO Success
Output execution 2
2019-11-07T07:08:19.270Z 3ce51f5d-bbbc-4dd6-b46f-2149ee9bb9cf INFO Success
Output execution 3
2019-11-07T07:08:27.410Z 2625bba5-b8e1-40e4-8704-7c0d486f6dff INFO Success
2019-11-07T07:08:27.431Z 2625bba5-b8e1-40e4-8704-7c0d486f6dff INFO Success
**
does anyone know the cause of this problem?
I am relatively new to node js 10.x. so please help me if I have missed something in code
**
you are using a callback and promise at the same time, remove the callback.
You can try something like
exports.handler = async (event, context) => {
const params = {
TableName: tableName,
Item: {
userid: event.userid,
}
};
try {
const data = await dynamoDB.put(params).promise();
console.log("Data: ", data);
} catch(error) {
console.error("Error:", error);
}
}

DynamoDB, Lambda function / custom module Timeout

I have the following two JS file. My problem is when i call the Calls.js which calls the Archive.js for archiving logs into DynamoDB the request times out.
I have tried out, many things, read about many things, tried in local/AWS environment without luck. What am i missing?
Link1, Link2, Link3, Link4, Link5,
Archive.js
module.exports.archive = archive;
...
function archive(input, callback){
AWS.config.update({
region: "eu-west-1",
endpoint: "http://localhost:8000"
});
var documentClient = new AWS.DynamoDB.DocumentClient({
httpOptions: {
agent: new https.Agent({
rejectUnauthorized: true,
secureProtocol: "TLSv1_method",
ciphers: "ALL"
})
}
});
...
var paramsPUT = {
TableName: "Logging",
Item: {
HashKey: dbID,
archiveEntry: archiveEntry
}
};
...
documentClient.put(paramsPUT, function(err, data) {
if (err) console.log(err);
if (data) console.log(data);
...
callback(data);
});
}
Calls.js
exports.handler(event, context, callback) => {
const archive = require("./..path..").archive;
...
context.callbackWaitsForEmptyEventLoop = false;
...
archive(input, callback);
...
}
I can not reproduce a timeout condition with your code. Your code is talking to an AWS endpoint at http://localhost:8000, so I assume you have DynamoDB local up and running, don't you ? Failling to have local DynamoDB running would cause the timeout.
That being said, I would strongly suggest to refactor your code to use Promise and the new async/await provided by NodeJS 8 instead of passing the Lambda callback around.
Here is the modified code.
const AWS = require("aws-sdk");
async function archive(input) {
return new Promise( (resolve, reject) => {
AWS.config.update({
region: "eu-west-1",
endpoint: 'http://localhost:8000'
});
//use client specific AWS configuration instead of the global one
const documentClient = new AWS.DynamoDB.DocumentClient();
var paramsPUT = {
TableName: "Logging",
Item: {
HashKey: "123",
archiveEntry: input
}
};
documentClient.put(paramsPUT, function (err, data) {
if (err) {
console.log("ERROR " + err);
reject(err);
}
console.log("Returned from DDB " + JSON.stringify(data, null,2));
resolve(data);
});
});
}
exports.handler = async (event, context, callback) => {
const result = await archive("abc");
callback(result);
}
// stuffs to test locally
callback = function (data) {
console.log("callback called with " + JSON.stringify(data,null,2));
}
event = context = {}
exports.handler(event, context, callback);

SQS to Lambda + SES

I'm new with Lambda & SQS and I'm trying to create a function to send emails, queued in an SQS service, but I don't understand how to call the process function that contains the send + delete queue methods.
Here bellow I paste my code:
'use strict';
const AWS = require('aws-sdk');
const SQS = new AWS.SQS({ apiVersion: '2012-11-05' });
const Lambda = new AWS.Lambda({ apiVersion: '2015-03-31' });
const ses = new AWS.SES({ accessKeyId: "xxxxxxxx", secretAccesskey: "xxxxxxx/xxxxxxxxx" });
const s3 = new AWS.S3({ apiVersion: "2006-03-01", region: "us-west-2" });
const QUEUE_URL = 'https://sqs.us-west-2.amazonaws.com/xxxxxxx/queue';
const PROCESS_MESSAGE = 'process-message';
function getPieceOfMail (path, mapObj, replace) {
return new Promise(function (resolve, reject) {
s3.getObject({
Bucket: "myBucket",
Key: "myKey/" + path
}, function (err, data) {
if (err) {
reject(err);
} else {
if (replace === true) {
var re = new RegExp(Object.keys(mapObj).join("|"), "gi");
data = data.Body.toString().replace(re, function (matched) {
return mapObj[matched.toLowerCase()];
});
resolve(data);
} else {
resolve(data.Body.toString());
}
}
});
});
}
function getRegisterSource (nickname, activate_link) {
var activate_link, pieces;
pieces = [
getPieceOfMail("starts/start.html", {}, false),
getPieceOfMail("headers/a.html", {}, false),
getPieceOfMail("footers/a.html", {}, false),
];
return Promise.all(pieces)
.then(function (data) {
return (data[0] + data[1] + data[2]);
})
.catch(function (err) {
return err;
});
}
function sendEmail (email, data) {
return new Promise(function (resolve, reject) {
var params = {
Destination: { ToAddresses: [email] },
Message: {
Body: {
Html: {
Data: data
},
Text: {
Data: data
}
},
Subject: {
Data: "myData"
}
},
Source: "someone <noreply#mydomain.co>",
};
ses.sendEmail(params, function (err, data) {
if (err) {
reject(err);
} else {
resolve(data);
}
});
});
}
function process(message, callback) {
console.log(message);
// process message
getRegisterSource(event['nickname'], event['user_id'])
.then(function (data) {
return sendEmail(event["email"], data);
})
.catch(function (err) {
console.log("==ERROR==");
callback(err, err);
})
.finally(function () {});
// delete message
const params = {
QueueUrl: QUEUE_URL,
ReceiptHandle: message.ReceiptHandle,
};
SQS.deleteMessage(params, (err) => callback(err, message));
}
function invokePoller(functionName, message) {
const payload = {
operation: PROCESS_MESSAGE,
message,
};
const params = {
FunctionName: functionName,
InvocationType: 'Event',
Payload: new Buffer(JSON.stringify(payload)),
};
return new Promise((resolve, reject) => {
Lambda.invoke(params, (err) => (err ? reject(err) : resolve()));
});
}
function poll(functionName, callback) {
const params = {
QueueUrl: QUEUE_URL,
MaxNumberOfMessages: 10,
VisibilityTimeout: 10,
};
// batch request messages
SQS.receiveMessage(params, (err, data) => {
if (err) {
return callback(err);
}
// for each message, reinvoke the function
const promises = data.Messages.map((message) => invokePoller(functionName, message));
// complete when all invocations have been made
Promise.all(promises).then(() => {
const result = `Messages received: ${data.Messages.length}`;
callback(null, result);
});
});
}
exports.handler = (event, context, callback) => {
try {
if (event.operation === PROCESS_MESSAGE) {
console.log("Invoked by poller");
process(event.message, callback);
} else {
console.log("invoked by schedule");
poll(context.functionName, callback);
}
} catch (err) {
callback(err);
}
};
can somebody throw me some light to this?
Thanks in advice.
UPDATE
After so much misconception, I've decided to start looking on how the example of polling-SQS works provided by AWS.
There I've found that I lacked some basic SQS permissions, but solved now by adding the right policy:
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": [
"lambda:InvokeFunction"
],
"Resource": ["*"]
}]
}
This allows Lambda.invoke() to call process().
When the process(message, callback) is called, if I console.log(message);, it seems that there's no message, although the queue is being cleared by the line SQS.deleteMessage(params, (err) => callback(err, message));
What I was trying was to combine my sendMail function that is currently working with a SQS service so I only have to push each message to the queue.
This is a common requirement where AWS SES has its own limitations in sending emails at once. If these limitations are violated, the SES account will sandbox itself. It seems like you have solved the problem using proper access credentials.
This code contains a Python3 Lambda code that can be used to handle a situation like this, where a Lambda polls from SQS using threading, and sends emails using SES, without exceeding the given limitations.
Link to Github Project.
You can also consider using the new feature in SQS, which is capable of invoking lambdas, when a new message is placed within SQS. But, be careful not to exceed the maximum number of lambda functions within the AWS Account region. (See this document)

Resources