I have some records in dynamoDB and trigger associated with it configured via serverless.yml file.
Below the configuration events:
- stream:
type: dynamodb
arn:
Fn::GetAtt:
- myTable
- StreamArn
batchSize: 1
But I have a requirement that based on some flag, I should stop the execution of all records (lambda) if one record ends in failure and subsequently all the records for that transaction has to be deleted from dynamoDB.
I noticed that even after deleting the records from DynamodB, the trigger still continues, Please is there a way to get the exisiting triggers with respect to the context and stop all ?
P.S I am using Nodejs
Code and Steps
Checker.js -- Talks to external system and adds the records to the dynamoDB table specified and exits. The below function Validate data is called via events from dynamodB. The serverless configuration for it is below
ValidateData :
handler: ValidateData.handler
memorySize: 1536
timeout: 300
events:
- stream:
type: dynamodb
arn:
Fn::GetAtt:
- kpiTaskTable
- StreamArn
batchSize: 1
ValidateData --
async.waterfall([
//go get kpis
function getKPIs(next) {
request({
agent: agent,
uri: getKPIsURL(endpoint, APIKey),
maxAttempts: retryCount,
retryDelay: retryDelayTime,
retryStrategy: request.RetryStrategies.HTTPOrNetworkError
}, function (error, response, body) {
if (error) {
console.log("ERROR:Error whilst fetching KPI's: " + error);
next(error);
} else {
//need to add in here to check that at least one kpi was returned otherwise error
kpis = JSON.parse(body);
if (kpis.constructor != Array) {
console.log("ERROR:Error KPI's are not of type Array");
next(error);
} else {
next(null);
}
}
});
}................................
function (err) {
if (err) {
console.log("ERROR: Something has gone wrong: " + err)
var stopReportGen = process.env.STOP_REPORT_CREATION_ON_ERROR;
if (stopReportGen === "true") {
console.log("Deleting records from dynamoDB for report ID " + reportId);
kpiUtil.deleteRecordsFromDynamoDB(reportId).then(function () {
s3Api.deleteFile(reportBucket, reportName, retryCount, retryDelayTime).then(function () {
console.log("INFO : The temp file is deleted from the S3 bucket")
callback(null, "ERROR: " + sourceId + "Report ID :" + reportId);
}).catch(function (err) {
console.log("ERROR : Error in deleting the temp file from the S3 bucket")
callback(null, "ERROR: " + sourceId + "Report ID :" + reportId);
})
});
}
Delete From Dynamodb -- Deleting the records from DB
var AWS = require('aws-sdk');
var fs = require('fs');
var path = require('path');
var zlib = require('zlib');
var fs = require('fs');
(function (exports) {
deleteRecordsFromDynamoDB = function (reportId) {
return new Promise(function (resolve, reject) {
var docClient = new AWS.DynamoDB.DocumentClient();
var table = process.env.KPI_TASK_TABLE;
var params = {
TableName: table,
FilterExpression: "#reportId = :reportId_val",
ExpressionAttributeNames: {
"#reportId": "reportId",
},
ExpressionAttributeValues: { ":reportId_val": parseInt(reportId) }
};
docClient.scan(params, onScan);
var count = 0;
function onScan(err, data) {
if (err) {
console.error("ERROR: Error, Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
reject(err);
} else {
console.log("Scan succeeded for reportID ::"+reportId);
data.Items.forEach(function (itemdata) {
var delParams = {
TableName: table,
Key: {
"reportSource": itemdata.reportSource
}
};
console.log("Attempting a conditional delete...");
docClient.delete(delParams, function (err, data) {
if (err) {
console.error("ERROR:Error, Unable to delete item. Error JSON:", JSON.stringify(err, null, 2));
reject(err);
} else {
console.log("DeleteItem succeeded:", JSON.stringify(data, null, 2));
}
});
console.log("INFO:Item :", ++count, JSON.stringify(itemdata));
});
// continue scanning if we have more items
if (typeof data.LastEvaluatedKey != "undefined") {
console.log("Scanning for more...");
params.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.scan(params, onScan);
}else{
resolve("sucess");
}
}
}
});
}
exports.deleteRecordsFromDynamoDB = deleteRecordsFromDynamoDB;
}(typeof exports === 'undefined' ? this['deleteRecordsFromDynamoDB'] = {} : exports))
Based on the above description, my understanding is that deleting the items will create the streams to lambda as well. You can ignore the delete streams in two ways:-
1) Check for eventName in Record. If the eventName is REMOVE, you can potentially ignore the stream in Lambda function
2) Before deleting the items in Dynamodb, please disable the stream on DynamoDB table using Update Table API.
Please note that Update Table is asynchronous operation. So it will take a while to reflect the change. The items should not be deleted until the stream is disabled. Otherwise, you can implement both option 1 and 2 to be in safer side.
var params = {
TableName: "SomeTableName",
StreamSpecification: {
StreamEnabled: false
}
};
dynamodb.updateTable(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data);
}
You may need to enable the stream when you would like to have the Lambda trigger back in operation.
Related
At the moment I have a function to get all items from a DynamoDB table using the SCAN option. This is an expensive way to do it and I would prefer using the QUERY option. But looking at the docs there does not seem to be a simple way to retrieve all items using the QUERY option - it expects some sort of condition.
Example
var params = {
TableName : "Movies",
KeyConditionExpression: "#yr = :yyyy",
ExpressionAttributeNames:{
"#yr": "year"
},
ExpressionAttributeValues: {
":yyyy": 1985
}
};
docClient.query(params, function(err, data) {
if (err) {
console.error("Unable to query. Error:", JSON.stringify(err, null, 2));
} else {
console.log("Query succeeded.");
data.Items.forEach(function(item) {
console.log(" -", item.year + ": " + item.title);
});
}
});
Expected
var params = {
TableName : "Movies"
};
docClient.query(params, function(err, data) {
if (err) {
console.error("Unable to query. Error:", JSON.stringify(err, null, 2));
} else {
console.log("Query succeeded.");
data.Items.forEach(function(item) {
console.log(" -", item.year + ": " + item.title);
});
}
});
Is it possible to retrieve all data from a table using QUERY? I thought of using BEGINS_WITH or such but all the primary keys are different/random and do not start with a specific character or phrase.
Technically, a query of all items in an Amazon DynamoDB table would return the same amount of data that a scan returns, so there should be no difference in cost.
The usual reduced efficiency of a scan operation is due to the fact that it has to read the whole table and then filters out values to provide the result you want, essentially adding the extra step of removing data from the result set. If you want to read the whole table without filtering, both scan and query have to retrieve all values and there is no additional filtering step.
The only way to do via query would be to loop over every partition key individually.
I'd suggest you look at a secondary index built around your query which will be more efficient: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/SecondaryIndexes.html
If you want to get all data you can use scan all data, but I recommend you to get data by limit and pagination because it can kill a lot of memory resources if you have millions of data at dynamodb.
this approach for getting all your data
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({
apiVersion: '2012-08-10',
region: 'ap-southeast-1' // put your region
});
exports.handler = async (event, context, callback) => {
const tableName = event.params.querystring.tablename;
let params = {
TableName: tableName
};
let scanResults = [];
let items;
do {
items = await docClient.scan(params).promise();
items.Items.forEach((item) => scanResults.push(item));
params.ExclusiveStartKey = items.LastEvaluatedKey;
} while (typeof items.LastEvaluatedKey != "undefined");
callback(null, scanResults);
};
But with the approach below, after you get data, you need to post the LastEvaluatedKey from the frontend to params and you can use it as ExclusiveStartKey.
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({
apiVersion: '2012-08-10',
region: 'ap-southeast-1' // put your region
});
exports.handler = async (event, context, callback) => {
const tableName = event.params.querystring.tablename;
let pageSize = event.params.querystring.pagesize;
let lastItem = event.params.querystring.lastItem;
try {
const params = {
TableName: tableName,
Limit: pageSize,
};
if (lastItem) {
params.ExclusiveStartKey = { id: lastItem};
}
const response = await docClient.scan(params).promise();
return {
items: response.Items,
lastItem: response.LastEvaluatedKey
};
} catch (error) {
throw error;
}
};
Using Lambda (node.js) - how to delete all the items in the Dynamodb table?
There are 500K rows in the table
I have tried using scan method and then loop through each item and then using delete method. It only allow up to 3000 rows only.
Code
exports.handler = function(context, callback) {
getRecords().then((data) => {
data.Items.forEach(function(item) {
deleteItem(item.Id).then((data1) => {
});
});
});
};
var deleteItem = function(id) {
var params = {
TableName: "TableName",
Key: {
"Id": id
},
};
return new Promise(function(resolve, reject) {
client.delete(params, function(err, data) {
if (err) {
reject(err);
} else {
resolve();
}
});
});
}
function getRecords() {
var params = {
TableName: 'TableName',
IndexName: 'Type-index',
KeyConditionExpression: 'Type = :ty',
ExpressionAttributeValues: {
':ty': "1"
},
ProjectionExpression: "Id",
};
return new Promise(function(resolve, reject) {
client.query(params, function(err, data) {
if (err) {
reject(err);
} else {
resolve(data);
}
});
});
}
There is already one right answer, but here is another code snippet to delete all records from Dynamo DB.
const AWS = require("aws-sdk");
AWS.config.update({
region: "us-east-1",
});
const docClient = new AWS.DynamoDB.DocumentClient();
const getAllRecords = async (table) => {
let params = {
TableName: table,
};
let items = [];
let data = await docClient.scan(params).promise();
items = [...items, ...data.Items];
while (typeof data.LastEvaluatedKey != "undefined") {
params.ExclusiveStartKey = data.LastEvaluatedKey;
data = await docClient.scan(params).promise();
items = [...items, ...data.Items];
}
return items;
};
const deleteItem = (table, id) => {
var params = {
TableName: table,
Key: {
id: id,
},
};
return new Promise(function (resolve, reject) {
docClient.delete(params, function (err, data) {
if (err) {
console.log("Error Deleting ", id,err);
reject(err);
} else {
console.log("Success Deleting ", id,err);
resolve();
}
});
});
};
exports.handler = async function (event, context, callback) {
try {
const tableName = "<table>";
// scan and get all items
const allRecords = await getAllRecords(tableName);
// delete one by one
for (const item of allRecords) {
await deleteItem(tableName, item.id);
}
callback(null, {
msg: "All records are deleted.",
});
} catch (e) {
callback(null, JSON.stringify(e, null, 2));
}
};
A Scan operation consumes Read capacity. Each Read returns up to 4 kb of data. When this limit is reached, the Scan returns only what it has found until there. If you need more, you need to issue another Scan request.
This, you'll need two loops: 1) loop to delete all records returned at each Scan; 2) loop to keep scanning multiple times, until you reach the end of the table
Make sure you use consistent Reads or wait 1 or 2 second(s) before issuing another Scan, otherwise you may get repeated items in different Scans.
exports.handler = function(context, callback) {
clearRecords();
};
clearRecords = function() {
getRecords().then((data) => {
data.Items.forEach(function(item) {
deleteItem(item.Id).then((data1) => {});
});
clearRecords(); // Will call the same function over and over
});
}
Observe that Lambda has a timeout limit of 15 minutes. Since you have 500K items in your table, it's likely that your Lambda will timeout and you'll need to trigger it more than once. You could also make your Lambda call itself after 14:50, for example, just take a look at the AWS SDK documentation for triggering Lambda functions. For this matter, you might also want to check the getRemainingTimeInMillis() method from the context object.
I seen aws document and i have insert parameter ReturnConsumedCapacity but not response ConsumedCapacity object... (use dynamodb local)
What was the mistake I made?
var params = {};
params.TableName = this.tableName;
params.Limit = 100;
params.ReturnConsumedCapacity = "TOTAL";
var items = [];
var scanExecute = function() {
dynamo.scan(params, function(err, result) {
console.log(result); // not found ConsumedCapacity...
if (err) {
console.log(err);
callback(new Error('dynamo scan error'));
} else if (result.Count === 0) {
callback(null, undefined);
} else {
items = items.concat(result.Items);
if (result.LastEvaluatedKey) {
params.ExclusiveStartKey = result.LastEvaluatedKey;
scanExecute();
} else {
callback(null, items);
}
}
});
}
scanExecute();
When you run the DynamoDB local instance, the consumed capacity will not be tracked.
Consumed capacity units are not tracked. In operation responses, nulls
are returned instead of capacity units.
I am new to dynamoDB and node.js I have written a code where it will make a query to the database (dynamodb) and look for an element which is entered by the user in the database. I am able to verify that but when the user tries with some other number which is not present in the database I am getting a null value.
My table name is "DevReferenceNumber" and only one column which is primary key "referencenumber".
'use strict';
var AWS = require('aws-sdk');
var docClient = new AWS.DynamoDB.DocumentClient({ region : 'us-east-1'});
function close(sessionAttributes, fulfillmentState, message) {
return {
sessionAttributes,
dialogAction: {
type: 'Close',
fulfillmentState,
message,
},
};
}
exports.handler = (event, context, callback) => {
try{
console.log(`event.bot.name=${event.bot.name}`);
if(event.bot.name != 'getCustomerReferenceNumber'){
callback('Invalid Bot Name');
}
dispatch(event, (response) => {
callback(null, response)
});
}catch(err){
callback("Error is occured while querying");
}
};
function dispatch(intentRequest, callback){
console.log(`dispatch UserID => ${intentRequest.userId}, intentName => ${intentRequest.currentIntent.name}`);
const intentName = intentRequest.currentIntent.name;
if(intentName === "checkReferenceNumber"){
return referenceNumber(intentRequest, callback);
}
}
function referenceNumber(intentRequest, callback){
const enteredReferenceNumber = intentRequest.currentIntent.slots.ReferenceNumber;
const sessionAttributes = intentRequest.sessionAttributes;
console.log("User has entered reference number is --> " + enteredReferenceNumber);
var params = {
TableName : "DevReferenceNumber",
KeyConditionExpression: "#refer = :ref",
ProjectionExpression : "#refer",
ExpressionAttributeNames: {
"#refer" : "referencenumber"
},
ExpressionAttributeValues:{
":ref": parseInt(enteredReferenceNumber)
}
};
docClient.query(params, function(err, data){
if(err){
callback(close(sessionAttributes, 'Fulfilled', {
contentType: 'PlainText',
content : 'Developer reference number is not matched with data from database'}));
}
else {
data.Items.forEach(function (item){
console.log("User matched data is ==> " + item.referencenumber);
callback(close(sessionAttributes, 'Fulfilled', {
contentType: 'PlainText',
content : 'Developer reference number is matched with data from database'}));
});
}
});
}
It is obvious that you will get a null record when you don't have a matching record. If you don't want null from node callback then you can do a custom logic to do a null check and return data according to the way you want.
I've a program that does the below.
Look into a DynamoDB table.
Get the data from the table.
Save the variables in session
After the process, print the values in console.
My code is as below.
intentHandlers['GetMYBusinessInfo'] = function (request, session, response, slots) {
console.log('entered ext bloxk');
if (!session.attributes.userName) {
console.log('eneterd the user entered the block');
var userName = 'jim';
isUserRegistered(userName.toLowerCase(), function (res, err) {
if (err) {
response.fail(err);
console.log(err);
}
else if (!res) {
response.shouldEndSession = true;
}
else {
console.log(res);
var countRes = JSON.stringify(res.Count);
var unpUserRegion = JSON.stringify(res.Items[0].Region);
var unpUserCity = JSON.stringify(res.Items[0].State);
var userRegion = JSON.parse(unpUserRegion);
var userCity = JSON.parse(unpUserCity);
session.attributes.city = userCity;
session.attributes.region = userRegion;
console.log("parsed " + countRes + "\t region is " + userRegion);
session.attributes.userName = true;
}
});
}
console.log(`session values after authentication are user city is ${session.attributes.city}`);
}
The method to check if the value is in DynamoDb or not.
function isUserRegistered(userName, callback) {
var params = {
TableName: "usersTable",
FilterExpression: "#nme = :nme",
ExpressionAttributeNames: {
"#nme": "Name",
},
ExpressionAttributeValues: {
":nme": userName
}
};
var count = 0;
docClient.scan(params, function (err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
callback(false, err);
} else {
console.log("Scan succeeded." + data.Items.length);
if (data.Items.length === 0) {
callback(false);
}
else {
data.Items.forEach(function (itemData) {
console.log("Item :", ++count, JSON.stringify(itemData));
});
callback(data);
}
}
});
}
when I run this, the output that I get is:
session values after authentication are user city is undefined
Scan succeeded.1
Item : 1
{
"ID": "3",
"State": "wisconsin",
"Region": "midwest",
"Name": "jim"
}
{ Items: [ { ID: '3', State: 'wisconsin', Region: 'midwest', Name: 'jim' } ],
Count: 1,
ScannedCount: 1 }
parsed 1 region is midwest
Here I know that Node js being Non-blockable process, the above output is correct, but I want to get the value of city printed in session values after authentication are user city is {HereTheCityComes} instead of session values after authentication are user city is undefined.
I'm sure that placing the console.log(session values after authentication are user city is ${session.attributes.city}); in the last else block(place where the data is returned).
But I need this type of functionality(Get data as shown in my current scenario), as there is some other things to be done after checking if the user is available in database.
please let me know where am I going wrong and how can I fix this.
You can't synchronously expect async result.
What you can do here is solve your problem with promises.
Here is a solution:
intentHandlers['GetMYBusinessInfo'] = function(request, session, response, slots) {
console.log('entered ext bloxk');
var userPromise = Promise.resolve();
if (!session.attributes.userName) {
console.log('eneterd the user entered the block');
var userName = 'jim';
userPromise = new Promise(function (resolve, reject) {
isUserRegistered(userName.toLowerCase(), function (res, err) {
if (err) {
response.fail(err);
reject(err);
}
var countRes = JSON.stringify(res.Count);
var unpUserRegion = JSON.stringify(res.Items[0].Region);
var unpUserCity = JSON.stringify(res.Items[0].State);
var userRegion = JSON.parse(unpUserRegion);
var userCity = JSON.parse(unpUserCity);
session.attributes.city = userCity;
session.attributes.region = userRegion;
console.log("parsed " + countRes + "\t region is " + userRegion);
resolve(res);
});
});
}
userPromise.then(function () {
console.log(`session values after authentication are user city is ${session.attributes.city}`);
});
}
If you are not using ES6, then just install bluebird and use var Promise = require('bluebird')