We have the following code used as lambda Function in Serverless Framework triggered every 2min with cron. The issue we are facing is that the writing in DynamoDB is inconsistent , we want to have 3 writings but instead we receive 1 or 2 writings every 2 minutes.
DynamoDB has a HASH key the HOUR and SORT KEY the DATE and Billing mode: PROVISIONED. Has someone faced the same behavior from DynamoDB or the same issue to share how he sovled it. Thanks
"use strict";
const AWS = require("aws-sdk");
const axios = require("axios");
const dynamoDb = new AWS.DynamoDB.DocumentClient();
const lambda = new AWS.Lambda({
region: "us-east-1",
});
module.exports.getWeather = async (event, context, callback) => {
const openWeatherMapAPIURL = `http://api.openweathermap.org/data/2.5/weather?id=${event}&appid=XXXXXXXXXXXXXXXXXXXXXXX&units=metric`;
const currentWeather = await axios
.get(openWeatherMapAPIURL)
.then((records) => {
console.log(records);
const d = new Date(records.headers.date);
let hour = d.getHours();
const params = {
TableName: process.env.DYNAMODB_TABLE_NAME,
Item: {
hour: hour,
date: records.headers.date,
city: records.data.name,
temp: records.data.main.temp,
feelsLike: records.data.main.feels_like,
description: records.data.weather[0].description,
},
};
setTimeout(function () {
dynamoDb.put(params, (error) => {
// handle potential errors
console.log(`zapis na: ${records.data.name} ${records.headers.date}`);
if (error) {
console.log(error);
console.error(error);
return;
}
});
}, 3000);
})
.catch((error) => {
console.log(error);
return;
});
const response = {
statusCode: 200,
body: JSON.stringify({
message: `Weather from ${event} was requested!`,
}),
};
callback(null, response);
};
module.exports.cron_launcher = (event, context, callback) => {
const requestedID = ["786735", "792578", "785842"];
requestedID.forEach((requestedID) => {
const params = {
FunctionName: process.env.HANDLER_LOCATION + "-getWeather",
InvocationType: "RequestResponse",
Payload: JSON.stringify(requestedID),
};
return lambda.invoke(params, function (error, data) {
if (error) {
console.error(JSON.stringify(error));
return new Error(`Error printing messages: ${JSON.stringify(error)}`);
} else if (data) {
console.log(data);
}
});
});
};
You are not waiting for the dynamodb.put operation to finish. Additionally, you are wrapping the call in a setTimeout. Your lambda function is returning before the network operation can be made. Make sure the put operation succeeds before returning a result from your lambda.
I see no reason for you to use a setTimeout here.
You can call dynamodb.put(...).promise() to get a promise from the dynamodb SDK and await that promise.
2.a Or you can continue using a callback, but wrap the entire section of code in a new promise object, calling the resolve method after the dynamodb.put call finishes.
Related
I am trying to mock SQS call defined in my file.js. It is a global instance in the file. So, while testing as I have to require file.js, its instance is set and my mock method is not called. However, if I set that SQS instance locally in the function inside which it is required, I am able to mock. But that would be wrong as that instance will be set every time that method is called. How can I mock SQS in my test? I have tried all the ways which were given there in the issues. None of them is working for me.
//file.js
const AWS = require('aws-sdk');
const sqs = new AWS.SQS();
const queueURL = config.sqs_connect.queue_url;
const params = {
MaxNumberOfMessages: 10,
QueueUrl: queueURL
};
exports.receiveMessages = async function () {
// let sqs = new AWS.SQS();
return new Promise((resolve, reject) => {
sqs.receiveMessage(params, function (err, data) {
if (err) {
console.log("error")
reject(err);
} else if (data.Messages) {
try {
consumeAndDeleteMessages(data.Messages, err => {
if (err) reject(err);
else resolve();
});
} catch (error) {
reject(error);
}
} else {
// logger.log("No data in queue");
resolve();
}
});
})
}
// file.test.js
const AWS = require('aws-sdk');
const consumer = require('path-to-file');
describe("foo", () => {
it("updates all info", async () => {
let delete_stack = [];
AWSMock.setSDKInstance(AWS);
AWSMock.mock('SQS', 'receiveMessage', (params, callback) => {
callback(null, { Messages: [{ MessageId: '1234', ReceiptHandle: 'qwertyu', Body: JSON.stringify(update_payload) }] });
});
AWSMock.mock('SQS', 'deleteMessageBatch', (params, callback) => {
delete_stack.push(params.Entries);
callback(null, {});
});
await consumer.receiveMessages();
AWSMock.restore('SQS');
expect(delete_stack).toStrictEqual([
[{ "Id": "1234", "ReceiptHandle": "qwertyu" }]
]);
});
});
If I define sqs locally in receiveMessage, the test will work file. I have tried all the ways provided, none of them is working. Am I doing something wrong?
I am writing node js 10.x lambda function to put details into DynamoDB table.
Below is code
const AWS = require('aws-sdk');
var db = new AWS.DynamoDB.DocumentClient();
var tableName="xyz";
exports.handler = async (event) => {
// TODO implement
console.log("Event: "+ JSON.stringify(event));
var response = {
statusCode: 200,
"headers": {
"Access-Control-Allow-Origin" : "*",
"Access-Control-Allow-Credentials" : true
},
};
await db.put({
TableName: tableName,
Item: {
userid: event.userid,
}
}, (error, data) => {
if (error) {
console.log("error:"+ error);
}
else{
console.log("Success");
}
}).promise();
return response;
};
I am getting kind on random number of success return
Output execution 1
2019-11-07T07:03:45.388Z f451dfc1-01ea-41d0-a998-945cb0f18be1 INFO Success
2019-11-07T07:03:45.510Z f451dfc1-01ea-41d0-a998-945cb0f18be1 INFO Success
2019-11-07T07:03:45.511Z f451dfc1-01ea-41d0-a998-945cb0f18be1 INFO Success
Output execution 2
2019-11-07T07:08:19.270Z 3ce51f5d-bbbc-4dd6-b46f-2149ee9bb9cf INFO Success
Output execution 3
2019-11-07T07:08:27.410Z 2625bba5-b8e1-40e4-8704-7c0d486f6dff INFO Success
2019-11-07T07:08:27.431Z 2625bba5-b8e1-40e4-8704-7c0d486f6dff INFO Success
**
does anyone know the cause of this problem?
I am relatively new to node js 10.x. so please help me if I have missed something in code
**
you are using a callback and promise at the same time, remove the callback.
You can try something like
exports.handler = async (event, context) => {
const params = {
TableName: tableName,
Item: {
userid: event.userid,
}
};
try {
const data = await dynamoDB.put(params).promise();
console.log("Data: ", data);
} catch(error) {
console.error("Error:", error);
}
}
I'm trying to put data into Dynamodb using serverless deployment. I have added the permission to write to Dynamodb.
The api is always sending {"message": "Internal server error"} but is able to put the data into db once if tried 5,6 times.
Following is the serverless.yaml config
handler: dynamoUpdate.handler
events:
- http:
path: /createdbentry
method: get
cors: true
Following is the code:
const AWS = require('aws-sdk')
AWS.config.update({ region: process.env.REGION || 'us-east-1' })
var ddb = new AWS.DynamoDB({apiVersion: '2012-08-10'});
exports.handler = async (event) => {
return await createDBEntry("i123","Working");
}
const sendRes = (status, body) => {
var response = {
statusCode: status,
headers: {
"Content-Type": "application/json"
},
body: body
};
return response;
};
const createDBEntry = async function(id,result) {
var params = {
TableName: 'emplist',
Item: {
'ID' : {S:id},
'Summary':{S: result},
}
};
ddb.putItem(params, function(err, data) {
console.log("Here comes me.")
if (err) {
console.log("Opps Error");
return sendRes(403,err);
} else {
console.log("Complete")
return sendRes(200,data);
}
});
}
How can it be resolved ?
The problem is you are not returning any promise or awaiting anything async in your function called createDBEntry. Thus your handler returns undefined which makes apigateway return 500 internal server error.
You are mixing callbacks and async/await.
Your createDBEntry function should look like this.
const createDBEntry = async function(id, result) {
var params = {
TableName: 'emplist',
Item: {
'ID' : {S:id},
'Summary':{S: result},
}
};
try {
let data = await ddb.putItem(params).promise()
return sendRes(200, JSON.stringify(data))
} catch (err) {
console.log("Oops Error");
return sendRes(403, err.toString());
}
}
When you return from an async handler in lambda the current execution environment is frozen immediately (unlike when you use a non-async handler with a callback). This is why most of the time the writes to the database does not succeed.
I have a lambda function written in node.js that returns a QRCode Image. I am also trying to read a value from the Dynamodb. However, the console logs inside it do not seem to be executed which makes me think the code is not being run.
I suspect this is due to so synchronization issues. But I am not sure what to do to fix it. The code is below:
var qrImage = require('qr-image');
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({region:'us-west-2'});
exports.handler = async(event, context, callback) => {
var path = event.path;
var drugId = path.replace(/\//g, '');
var params = {
TableName: 'QRCodeInfo',
Key: {
"DrugId" : "1234"
}
};
docClient.get(params, function(err,data) { //does not get executed
if (err) {
console.log(err);
} else {
console.log(data);
}
});
return sendRes(200,drugId); //this works. Image is seen.
};
const sendRes = (status, body) => {
//console.log(body);
const svg_string = qrImage.imageSync(body, { type: 'svg', size: 10 });
var response = {
statusCode: status,
headers: {
"Content-Type": "image/svg+xml"
},
body: svg_string
};
return response;
};
You are probably exiting the lambda before the callback of the dynamodb call has had a chance to execute.
Try calling callback(null, data) in the callback of the dynamo call, after your console.log and similar in the err scenario e.g. callback(err)
You do not exit a lambda by calling return, you should be calling callback() (that's why it's available as the 3rd argument of the lambda) see https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html#nodejs-prog-model-handler-callback
I have the following two JS file. My problem is when i call the Calls.js which calls the Archive.js for archiving logs into DynamoDB the request times out.
I have tried out, many things, read about many things, tried in local/AWS environment without luck. What am i missing?
Link1, Link2, Link3, Link4, Link5,
Archive.js
module.exports.archive = archive;
...
function archive(input, callback){
AWS.config.update({
region: "eu-west-1",
endpoint: "http://localhost:8000"
});
var documentClient = new AWS.DynamoDB.DocumentClient({
httpOptions: {
agent: new https.Agent({
rejectUnauthorized: true,
secureProtocol: "TLSv1_method",
ciphers: "ALL"
})
}
});
...
var paramsPUT = {
TableName: "Logging",
Item: {
HashKey: dbID,
archiveEntry: archiveEntry
}
};
...
documentClient.put(paramsPUT, function(err, data) {
if (err) console.log(err);
if (data) console.log(data);
...
callback(data);
});
}
Calls.js
exports.handler(event, context, callback) => {
const archive = require("./..path..").archive;
...
context.callbackWaitsForEmptyEventLoop = false;
...
archive(input, callback);
...
}
I can not reproduce a timeout condition with your code. Your code is talking to an AWS endpoint at http://localhost:8000, so I assume you have DynamoDB local up and running, don't you ? Failling to have local DynamoDB running would cause the timeout.
That being said, I would strongly suggest to refactor your code to use Promise and the new async/await provided by NodeJS 8 instead of passing the Lambda callback around.
Here is the modified code.
const AWS = require("aws-sdk");
async function archive(input) {
return new Promise( (resolve, reject) => {
AWS.config.update({
region: "eu-west-1",
endpoint: 'http://localhost:8000'
});
//use client specific AWS configuration instead of the global one
const documentClient = new AWS.DynamoDB.DocumentClient();
var paramsPUT = {
TableName: "Logging",
Item: {
HashKey: "123",
archiveEntry: input
}
};
documentClient.put(paramsPUT, function (err, data) {
if (err) {
console.log("ERROR " + err);
reject(err);
}
console.log("Returned from DDB " + JSON.stringify(data, null,2));
resolve(data);
});
});
}
exports.handler = async (event, context, callback) => {
const result = await archive("abc");
callback(result);
}
// stuffs to test locally
callback = function (data) {
console.log("callback called with " + JSON.stringify(data,null,2));
}
event = context = {}
exports.handler(event, context, callback);