I have created a Lambda function which puts an Item in the DynamoDB table using Node.js 10.x. The Lambda function has been integrated with SQS Queue via trigger. When I put a message in the Queue the Lambda is invoked, but is throwing an error while inserting an Item in the DynamoDB table.
Message -> SQS -> Lambda -> DynamoDB
I am passing the TableName and the Item as the input to the DynamoDB put method. But, the CloudWatch Logs says that TableName and Item parameters are missing. Below are the code snippets and the screenshots.
I did spend a good amount of time, but not able to get around the problem. Could someone help me?
Lambda Code
console.log('Loading function');
var AWS = require('aws-sdk');
var dynamo = new AWS.DynamoDB.DocumentClient();
exports.handler = function(event, context, callback) {
console.log('Received event:', JSON.stringify(event, null, 2));
event.Records.forEach(record => {
const { body } = record;
console.log("Input to DynamoAPI :", body);
dynamo.put(body, callback);
});
return {};
}
Message that goes into the Queue
{
"TableName": "user-transaction",
"Item" : {
"transaction-id": 1
}
}
Error message from the CloudWatch
I did not try this out, but it looks to me like there is a missing JSON.parse() around the body variable in the dynamo put invocation.
From what I remember, SQS stringifies the body and it is not parsed automatically.
The following answer which I found on SO sums this up perfectly: [1].
References
[1] https://stackoverflow.com/a/56925968/10473469
I checked a similar lambda of my own thats successfully doing a put.
var params = {
TableName: dynamoUserTableName,
Item:{
"userId" : { S: event.request.userAttributes.sub},
"email" : { S: event.request.userAttributes.email},
"userName" : { S: event.userName},
"createdDate" : { S: createdDate },
"version" : { N: '1'}
}
};
console.log(params);
dynamodb.putItem(params, function(err, data) {...
Here is the output from my console just before the put
{ TableName: 'Users',
Item:
{ userId: { S: 'xxxxxxxxxxxxxxxxxxxxxxxxx' },
email: { S: 'test#example.co.uk' },
userName: { S: 'xxxxxxxxxxxxxxxxxxxxxxxx' },
createdDate: { S: '2020-05-02T07:49:23.478' },
version: { N: '1' } } }
I note there are no quotes around my attribute names, maybe this is an issue?
Related
Here is my data structure -
{
"pk": "USER-12560000000",
"sk": "USER",
"data": {
"name": "George", //I want to modify the value of this attribute
"updatedAt": Date.now() // I want to add this attribute to the data field
}
}
I have been trying this below updateExpression, but for some reason it doesn't even give a clue on whats wrong in cloudwatch. In the below code I am only trying to update the value of the displayName attribute, not sure on the code on how to add an extra attribute. The cloudwatch logs dont print the error reason as well. I have read the documentation on aws also a few other stackoveflow posts. I haven't been able to make this work.
Main Lambda Handler -
const updateUser = require('./user-queries/updateUser');
exports.handler = async (event) => {
var userId = event.arguments.userId;
var name = event.arguments.name;
var avatarUrl = event.arguments.avatarUrl;
console.log(event.info.fieldName);
switch(event.info.fieldName) {
case "updateUser":
return updateUser(userId, name, avatarUrl);
}
};
const AWS = require("aws-sdk")
AWS.config.update({ region: "ca-central-1" })
const dynamoDB = new AWS.DynamoDB.DocumentClient()
async function updateUser(userId, name, avatarUrl) {
dynamoDB
.update({
TableName: "Bol-Table",
Key: {
pk: "USER-12560000000",
sk: "USER",
},
UpdateExpression: `set data = :data`,
ExpressionAttributeValues: {
":data": {
"name": "Test"
},
},
})
.promise()
.then(data => console.log(data.Attributes))
.catch(console.error)
}
module.exports = updateUser;
This is what gets printed out in cloudwatch along with some billing information
2021-10-09T16:25:15.527Z b1a1e4cb-6001-415c-82a3-cfdc90413e4e INFO USER-12560000000 Sam www.bol.com
Your UpdateExpression & ExpressionAttributeValues are wrong, they should be:
UpdateExpression: `set data.name = :x`
...
ExpressionAttributeValues: {
":x": "Test"
}
The Key object also takes in the keys as a string so your Key object needs to look like:
Key: {
"pk": "USER-12560000000",
"sk": "USER"
}
To update/insert multiple fields, the below will work:
UpdateExpression: `set data.name = :x, data.updatedAt = :y`
...
ExpressionAttributeValues: {
":x": "Test",
":y": Date.now()
}
Changing the dynamodb.update like below made it all work after all.
await dynamoDB.update(params).promise();
Here is the full code for anyone who comes across this issue.
const AWS = require("aws-sdk")
AWS.config.update({ region: "ca-central-1" })
const dynamoDB = new AWS.DynamoDB.DocumentClient()
async function updateUser(userId, name, avatarUrl) {
var params = {
TableName: "Bol-Table",
Key: {
"pk": userId,
"sk": 'USER',
},
UpdateExpression: 'SET details.displayName = :name, details.avatarUrl = :avatarUrl, details.updatedAt = :updateDate',
ExpressionAttributeValues: {
':name': name,
':avatarUrl':avatarUrl,
':updateDate': new Date().toISOString()
},
ReturnValues: 'ALL_NEW'
};
const Item = await dynamoDB.update(params).promise();
console.log(Item)
}
module.exports = updateUser;
I've hit a weird problem where dynamodb.getItem returns {} when I specify a parameter that ends with ID:
var AWS = require("aws-sdk");
AWS.config.update({ region: "ap-southeast-2" });
var dynamodb = new AWS.DynamoDB({ apiVersion: '2012-08-10' });
const tableName = "ReportingDB"
async function logReports() {
try {
var params = {
Key: {
"Category": { "S": "Certification" },
"Report": { "S": "By EmployeeID" }
},
TableName: tableName
};
var result = await dynamodb.getItem(params).promise()
console.log(JSON.stringify(result))
} catch (error) {
console.error(error);
}
}
When I change the value "By EmployeeID" to "By Employee Number" it then works, why?
async function logReports() {
try {
var params = {
Key: {
"Category": { "S": "Certification" },
"Report": { "S": "By Employee Number" }
},
TableName: tableName
};
var result = await dynamodb.getItem(params).promise()
console.log(JSON.stringify(result))
} catch (error) {
console.error(error);
}
}
To reproduce, create a DynamoDB table with two columns Category and Report, then add an item with "Certification" and "By EmployeeID/Number" values respectively.
The error handling in DynamoDB is pretty good, typically it will fail with an error message such as Reserved keyword (eg Using a ProjectionExpression with reserved words with Boto3 in DynamoDB) however with "By EmployeeID" it doesn't produce any error, it just doesn't return anything. Could it be a bug or something I don't know about? I couldn't find it documented.
PS I've inserted the values in the dynamo db using the web console.
The documentation for GetItem explains that:
If there is no matching item, GetItem does not return any data and there will be no Item element in the response.
In other words, if there is no item matching the given key, you'll get an empty result - not an error message. I suspect that this is exactly what is happening in your case.
I don't know why it didn't find an item with "By EmployeeID". I don't think it's a bug involving the letters "ID" :-) I suspect that you simply have a bug in the code which inserted this item, perhaps mis-spelling the word "By EmployeeID". Please look again at the code which inserts this item, or use the AWS DynamoDB UI (or do a Scan request) to inspect the contents of your table manually to see what it actually contains.
I am using nodejs aws-sdk/clients/dynamodb library with dynamodb. I need to update an item in the table. Below is the sample code to update an item:
params = {
TableName:table,
Key:{
"year": year,
"title": title
},
UpdateExpression: "set info.rating = info.rating + :val",
ExpressionAttributeValues:{
":val": 1
},
ReturnValues:"UPDATED_NEW"
};
I will have to specify each attribute in info in UpdateExpression. My info object is very big and I am looking for an easier way to do that. Is there a build-in method to support update an object to dynamodb item? something like:
params = {
TableName:table,
Key:{
"year": year,
"title": title
},
Item: info
};
The answer given by E.J. Brennan is great for cases where it's ok to replace the entire item. DocumentClient eases the hassle of dealing with DynamoDB attribute types, but the example given uses the put method. According to the docs put passes through to putItem which
Creates a new item, or replaces an old item with a new item
That means that it's not going to help with partial updates to existing items where you don't already have the full record (and can get away with a full replacement). For partial updates you have to use updateItem, or it's DocumentClient counterpart, update.
The AWS labs has published a utility to help with constructing update expressions to use with updateItem. Since I generally prefer to use DocumentClient, I unmarshall values with the utility function provided by DynamoDB's Converter (yes, I know it's a bit a back and forth, but it makes testing easier).
const AWS = require('aws-sdk');
const db = new AWS.DynamoDB.DocumentClient();
const { UpdateExpression, ExpressionAttributes } = require('#aws/dynamodb-expressions');
const { unmarshall } = AWS.DynamoDB.Converter;
const updateExpressionProps = ({ category, classification }) => {
attributes = new ExpressionAttributes();
expression = new UpdateExpression();
expression.set('category', category);
expression.set('classification', classification);
return {
UpdateExpression: expression.serialize(attributes),
ExpressionAttributeNames: attributes.names,
ExpressionAttributeValues: unmarshall(attributes.values),
};
};
const updateRequest = async ({ id, subject, category, classification }) =>
await db
.update({
TableName: 'table-name',
Key: {
id,
subject,
},
...updateExpressionProps({ category, classification }),
})
.promise();
This bit of code only updates the category and classification attributes on the record identified with id and subject without the hassle of manually building a correct UpdateExpression string. This example could easily be generalized into something reusable throughout your project.
I wrote the following utility method to convert a given object to UpdateExpression (SET only), ExpressionAttributeNames, and ExpressionAttributeValues.
const convertToCompositePathObj = (obj: Record<any, any>) => {
const res: Record<string, string | number | []> = {};
const getPropertyPath = (obj: Record<any, any>, current = "") => {
for (let key in obj) {
const value = obj[key];
const newKey = current ? [current, key].join(".") : key;
if (value && typeof value === "object" && !Array.isArray(value)) {
getPropertyPath(value, newKey);
} else res[newKey] = value;
}
};
getPropertyPath(sampleObject);
return res;
};
const generateDynamoDbUpdateExpression = (obj: Record<any, any>) => {
const compositePathObj = convertToCompositePathObj(sampleObject);
let counter = 0;
let updateExpression = "SET ";
const expressionAttNamesMap: any = {};
const expressionAttValuesMap: any = {};
for (let k in compositePathObj) {
const newUpdateExpression = k
.split(".")
.map((item) => {
const attName = `#${item}`;
if (!expressionAttNamesMap[attName]) {
expressionAttNamesMap[attName] = item;
}
return attName;
})
.join(".")
.concat(`= :${counter} AND `);
expressionAttValuesMap[`:${counter}`] = compositePathObj[k];
counter += 1;
updateExpression += newUpdateExpression;
}
updateExpression = updateExpression.substring(0, updateExpression.length - 5);
return {
UpdateExpression: updateExpression,
ExpressionAttributeNames: expressionAttNamesMap,
ExpressionAttributeValues: expressionAttValuesMap
};
};
// example usage:
const sampleObject = {
name: {
first: "John",
last: "Doe"
},
address: {
line1: "123 test st.",
line2: "Apt 123",
city: "Los Angeless",
state: "CA",
zip: 92763
},
phone: 8675768475
};
console.log(generateDynamoDbUpdateExpression(sampleObject));
/**
*****OUTPUT*****
{
UpdateExpression: "SET #name.#first= :0 AND #name.#last= :1 AND #address.#line1= :2 AND #address.#line2= :3 AND #address.#city= :4 AND #address.#state= :5 AND #address.#zip= :6 AND #phone= :7",
ExpressionAttributeNames:{
#name: "name"
#first: "first"
#last: "last"
#address: "address"
#line1: "line1"
#line2: "line2"
#city: "city"
#state: "state"
#zip: "zip"
#phone: "phone"
},
ExpressionAttributeValues:{
:0: "John"
:1: "Doe"
:2: "123 test st."
:3: "Apt 123"
:4: "Los Angeless"
:5: "CA"
:6: 92763
:7: 8675768475
}
}
**/
PS: I wrote this in a hurry so please excuse the formatting and any types.
https://codesandbox.io/s/aws-dynamic-update-expression-set-33tye7?file=/src/index.ts
You could use the Document Client:
Version 2.2.0 of the AWS SDK for JavaScript introduces support for the
document client abstraction in the AWS.DynamoDB namespace. The
document client abstraction makes it easier to read and write data to
Amazon DynamoDB with the AWS SDK for JavaScript. Now you can use
native JavaScript objects without annotating them as AttributeValue
types.
For example:
var docClient = new AWS.DynamoDB.DocumentClient({region: 'us-west-2'});
var params = {
Item: {
hashkey: 'key',
boolAttr: true,
listAttr: [1, 'baz', true]
mapAttr: {
foo: 'bar'
}
},
TableName: 'table'
};
docClient.put(params, function(err, data){
if (err) console.log(err);
else console.log(data);
});
https://aws.amazon.com/blogs/developer/announcing-the-amazon-dynamodb-document-client-in-the-aws-sdk-for-javascript/
I'm trying to return only values where john is found from a DynamoDB database.
I'm able to return values where it contains name: john from a mapped list, however the problem am having is that it appears to also be returning other values as well.
Running select: 'count' returns 1 match which is correct but it doesn't return anything when used.
I'm assuming that count just returns a number and not a specific select where john is matched.
I'm writing this in NodeJS; am hoping someone can help me figure this out.
I know that the value I only want shown are json elements where name: john, anything else I want omitted from being shown.
Here's my result as of right now:
{
"Department": [
{
"employees": [
{
"name": "john"
},
{
"name": "sally"
}
]
}
],
"Count": 1
}
My code:
const AWS = require('aws-sdk'); // eslint-disable-line import/no-extraneous-dependencies
const dc = new AWS.DynamoDB.DocumentClient();
exports.handler = async (event, context, callback) => {
// Construct the params for filtering data through dynamodb
const params = {
FilterExpression: "contains(#department, :employees)",
ExpressionAttributeNames: {
"#department": "employees",
},
ExpressionAttributeValues: {
":employees": {
"name":"john"
}
},
ProjectionExpression: "#department",
TableName: 'mytable',
//Select: 'COUNT'
};
const resultDC = await dc.scan(params).promise();
var items = resultDC.Items;
var count = resultDC.Count;
var returned_list = {
'Department' : items,
'Count' : count,
};
// create a response
const response = {
statusCode: 200,
body: JSON.stringify(returned_list),
};
callback(null, response);
};
I suggest you to use a Local Secondary Index.
Take a look here.
I am trying to retrieve values from DynamoDB when passing the primary partition key as the parameters in the URL.
I have a table with two columns:
movie id
movie name.
I want to be able to pass the movie id in the URL and to get the correspondent movie as a result. I have already created following lambda function and connected it to an API and DynamoDB.
I am not able to figure out the piece of code to use in order to retrieve the movie name when passing movie id as a parameter
Here is my code so far:
console.log('Loading function');
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({region: 'us-east-2'});
exports.handler = function(event, context,callback) {
var params = {
TableName: 'movie',
Key: {
"movieID": 5,
}
}
docClient.get(params, function(err, data){
if(err){
callback(err,null);
}else{
callback(null, data);
}
})
};
Try
var params = {
TableName: 'movie',
Key: {
"movieID": 5,
},
AttributesToGet= [
"movie_name"
],
ProjectionExpression: 'movie_name"
}
https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/DynamoDB.html
AttributesToGet is a legacy field as per documentation and ProjectionExpression should be used instead.