Query a Global Secondary Index using contains in DynamoDB local - node.js

I have id as the hash key of my table and returnItemId which is the GSI. The returnItemId is a string which contains values separated by commas. Given a number for the GSI, I want to be able to query and get the correct item that contains it by using contains
var params = {
"AttributeDefinitions": [ // describbes the key schema of the table
{
"AttributeName": "id",
"AttributeType": "S"
},
{
"AttributeName": "returnItemId",
"AttributeType": "S"
}
],
// Hash for Primary Table
"KeySchema": [
{
"AttributeName": "id",
"KeyType": "HASH"
}
],
"GlobalSecondaryIndexes": [
{
"IndexName": "ReturnItemIndex",
"KeySchema": [
{
"AttributeName": "returnItemId", //must match one of attributedefinitions names
"KeyType": "HASH"
}
],
"Projection": {
"ProjectionType": "ALL"
},
"ProvisionedThroughput": {
"ReadCapacityUnits": 5,
"WriteCapacityUnits": 5
}
}
],
"ProvisionedThroughput": {
"ReadCapacityUnits": 5,
"WriteCapacityUnits": 5
},
"TableName": "my-table"
};
dynamodb.createTable(params, function(err, data) {
if (err) ppJson(err); // an error occurred
else ppJson(data); // successful response
});
Then I am going to create 2 items
var params = {
TableName: 'my-table',
Item: {
"id": "the_first_item",
"returnItemId": "123,456,789"
},
};
docClient.put(params, function(err, data) {
if (err) ppJson(err); // an error occurred
else ppJson(data); // successful response
});
And the second item
var params = {
TableName: 'my-table',
Item: {
"id": "the_second_item",
"returnItemId": "987,654,321"
},
};
docClient.put(params, function(err, data) {
if (err) ppJson(err); // an error occurred
else ppJson(data); // successful response
});
The two items look like
I am trying to run a query and get the correct item which contains 987 using the following query. Since my first item has 123,456,789 and the second item has 987,654,321 this method should return the second item.
var params = {
TableName: 'my-table',
IndexName: 'ReturnItemIndex', // optional (if querying an index)
KeyConditionExpression: 'contains(returnItemId, :return_id)',
//FilterExpression: 'contains(returnItemId, :return_id)', // a string representing a constraint on the attribute
ExpressionAttributeValues: { ':return_id': '987' },
};
docClient.query(params, function(err, data) {
if (err) ppJson(err); // an error occurred
else ppJson(data); // successful response
});
But am getting errors about using contains in keyconditionexpression. Is this method possible?

contains can only be used in filters which means:
- query or scan operations will traverse all data to apply your filters
- your cost of read operations will include all data read not just matched data
- with contains "12", you would probably match "123" and "124" too
- better than comma separated is to use StringSet or NumberSet data type
I would suggest another layout
Keyschema:
Partiton Key: id
Sort Key: returnItemId
GSI
Partition Key: returnItemId
Data:
------------------------------------
| id | returnItemId |
------------------------------------
| "the_first_item" | "123" |
| "the_first_item" | "456" |
| "the_first_item" | "789" |
| "the_second_item" | "987" |
| "the_second_item" | "654" |
| "the_second_item" | "321" |
------------------------------------
then query GSI for key condition returnItemId = 987 ( no filter expression )

Related

Lambda - Querying DynamoDB through Lambda

I have a table called Customers with attributes CustId(partition Key), Fname, Lname, Dob.
I created a secondary index called LastNameIndex on Lname with the following params:
{
TableName: 'Customers'
AttributeDefinitions: [
{
AttributeName: 'Lname',
AttributeType: 'S'
}
],
GlobalSecondaryIndexUpdates: [
{
Create: {
IndexName: "LastNameIndex",
KeySchema: [
{AttributeName: "Lname", KeyType: "HASH"}
],
Projection: {
"ProjectionType": "ALL"
},
ProvisionedThroughput: {
"ReadCapacityUnits": 1,"WriteCapacityUnits": 1
}
}
}
]
}
Lambda function (snippet) - I want to get all records with Lname=Connors
params = {
TableName: "Customers",
IndexName: "LastNameIndex",
ExpressionAttributeNames: {
"#FN": "Fname",
"#LN": "Lname",
"#DB": "Dob",
},
ExpressionAttributeValues: {
":a": {
S: "Connors"
}
},
KeyConditionExpression: "Lname = :a",
ProjectionExpression: "#FN, #LN, #DB"
};
Running the query
ddb.query(params).promise().then(
function(data) {
console.log("Customer:" + data.Item)
return data.Item;
},
function() {
console.log("No records found...")
}
);
I have a record with Lname = Connors.
But the query does not return me any records - any idea what's wrong with the params?
The query operation returns multiple items not a single item like getItem operation. so I think returning data.Items instead of data.Item should work just fine.
ddb.query(params).promise().then(
function(data) {
console.log("Customer:" + data.Items)
return data.Items;
},
function() {
console.log("No records found...")
}
);

How to Append value to an empty list attribute in AWS DynamodB

I am trying to append a message to an empty list in AWS DynamoDB.
Here is the error that I get when I run the function
Invalid UpdateExpression: Incorrect number of operands for operator
or function; operator or function: list_append, number of operands: 1
Data structure in DynamodB
Below is the code:
var caseId = "1734009";
var chatMessage = { "2018-04-20T15:02:48Z":
{
"userId": "wQnUJrklzwWBDOsx83XVETSS7us2",
"message": "How are you"
}
}
var params = {
TableName : 'CHATS',
Key: {
"CASE_ID" : caseId
},
UpdateExpression : "SET CHAT_MESSAGES = list_append(:i)",
ExpressionAttributeValues : {
':i': [chatMessage],
},
ReturnValues:'UPDATED_NEW' // OTHER OPTIONS: NONE | ALL_OLD | UPDATED_OLD | ALL_NEW | UPDATED_NEW
};
documentClient.update(params, function(err, data) {
if(err) {
var message = "Chat message could not be saved, error:" + JSON.stringify(err, null, 2);
res.json({"status": "failure", "statusCode" : 400, "message": message});
} else {
next();
}
});
list_append takes two arguments, not one: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.UpdateExpressions.html
By using the below params and referring to the AWS Docs provided by Jason Livesay, I was able to add items to an empty list.
var params = {
TableName : 'CHATS',
Key: {
"CASE_ID" : caseId
},
UpdateExpression : "SET #ri = list_append(#ri, :vals)",
ExpressionAttributeNames: {
"#ri": "CHAT_MESSAGES"
},
ExpressionAttributeValues : {
':vals': [chatMessage]
},
ReturnValues:'UPDATED_NEW' // OTHER OPTIONS: NONE | ALL_OLD | UPDATED_OLD | ALL_NEW | UPDATED_NEW
};
Look at the following example. Attribute 'graph' is created as empty list which can be later updated using list_append(). Please note I am using Boto3(python) to update my dynamodb table.
dynamoTable = dynamodb.Table('abc')
dynamoTable.put_item(
Item={
''<primary_key>'': <primary_key_value>,
'First Name': 'first_name',
'Last Name': 'last_name',
'password_hash': 'password_hash',
'salt': 'salt',
'graph': [],
}
Look at the following code to append element/message to the list :
dynamoTable = dynamodb.Table('abc')
dynamoTable.update_item(
Key={
'<primary_key>': '<primary_key_value>',
},
UpdateExpression="set #Graph = list_append(#Graph, :name)",
ExpressionAttributeNames={
'#Graph': 'graph',
},
ExpressionAttributeValues = {
':name': [# {'myObject':
{
"userId": "wQnUJrklzwWBDOsx83XVETSS7us2",
"message": "How are you"
}
#}
],
}
The code above will append a map in 'graph' with the "userId" and "message"

The number of conditions on the keys is invalid dynamo db with node js

docClient.update({
TableName: 'patient',
Key: {
"patientId": "TIGERPAT0001"
},
UpdateExpression: "set title = :x, name = :y",
ExpressionAttributeNames: {
"#name": "name"
},
ExpressionAttributeValues: {
":x": 'title value abc',
":y": 'name value xyz'
}
}, function (err, data) {
if (err) {
json.status = '0';
json.result = { 'error': 'Unable to Edit Patient : ' + JSON.stringify(err) };
res.send(json);
} else {
json.status = '1';
json.result = { 'sucess': 'Patient Edited Successfully :' };
res.send(json);
}
});
when use above code, i got res :
Unable to Edit Patient Error : `{"message":"The number of conditions on the keys is invalid",
"code":"ValidationException",
"time":"2017-09-13T07:12:56.608Z",
"requestId":"a01c707c-86b4-41a5-a1c5-92b9ea07c026",
"statusCode":400,"retryable":false,
"retryDelay":6.368631970657979}`
What do I miss / any mistake??
I think you have used multiple keys while creating table.
If you have used n number of keys while creating table, then here also you need to pass n number of keys.
Note below, we are passing in id1 and id2 keys as well.
Ex:
docClient.update({
TableName: 'patient',
Key: {
"patientId": "TIGERPAT0001",
"id1": "id1value",
"id2": "id2value"
},
UpdateExpression: "set title = :x, #name = :y",
ExpressionAttributeNames: {
"#name": "name"
},
ExpressionAttributeValues: {
":x": 'title value abc',
":y": 'name value xyz'
}
}, function (err, data) {
if (err) {
json.status = '0';
json.result = { 'error': 'Unable to Edit Patient : ' + JSON.stringify(err) };
res.send(json);
} else {
json.status = '1';
json.result = { 'sucess': 'Patient Edited Successfully :' };
res.send(json);
}
});
Please replace id1 and id2 with your keys

node.js passing a parameter to DynamoDB updateItem method

I want to write a function that updates given parameter in dynamodb.
For example in a dynamodb table where each userId is the key I have values like
{
"categoryname": "a",
"skillState": "a",
"skipcount": 1,
"userId": "amzn1.ask.account.xxx”
}
I wanna set the "categoryname": "b" although there might be 10-15 fields like this so I dont wanna hard code the field name.
function (userId,itemToUpdate,itemValue,callback) {
var updateExpressionString = "SET #"+itemToUpdate+" =:val1";
var expressionAtt = '#'+itemToUpdate + '';
console.log(updateExpressionString)
console.log(expressionAtt)
this.dynamodb.updateItem({
TableName: constants.dynamoDBDetailTableName,
Key: {
userId: {
S: userId
}
},
UpdateExpression: updateExpressionString,
ExpressionAttributeNames : {
expressionAtt : itemToUpdate
},
ExpressionAttributeValues : {
':val1': {'S':itemValue}
}
}, function (err, data) {
if (err) {
console.log(err)
console.log('Error ')
} else if (data.Item === undefined) {
}else {
console.log(data)
}
});
}
In ExpressionAttributeNames:
{ ValidationException: ExpressionAttributeNames contains invalid key: Syntax error; key: "expressionAtt"
This throws error obviously thinking that expressionAtt is the key while it is a local variable.
I am new to node.js , how can pass the local variable in to ExpressionAttributeNames and ExpressionAttributeValues
One way of dealing with this could be to pull the object out of updateItem, put it into its own variable like so:
var item = {
TableName: constants.dynamoDBDetailTableName,
Key: {
userId: {
S: userId
}
},
UpdateExpression: updateExpressionString,
ExpressionAttributeNames: {},
ExpressionAttributeValue: {
':val1': {'S': itemValue}
}
};
item.ExpressionAttributeNames[expressionAtt] = itemToUpdate;
this.dynamodb.updateItem(item);
I believe that will fix your problem

How do you query for a non-existent (null) attribute in DynamoDB

I'm trying to query a DynamoDB table to find all items where the email attribute is not set. A global secondary index called EmailPasswordIndex exists on the table which includes the email field.
var params = {
"TableName": "Accounts",
"IndexName": "EmailPasswordIndex",
"KeyConditionExpression": "email = NULL",
};
dynamodb.query(params, function(err, data) {
if (err)
console.log(JSON.stringify(err, null, 2));
else
console.log(JSON.stringify(data, null, 2));
});
Result:
{
"message": "Invalid KeyConditionExpression: Attribute name is a reserved keyword; reserved keyword: NULL",
"code": "ValidationException",
"time": "2015-12-18T05:33:00.356Z",
"statusCode": 400,
"retryable": false
}
Table definition:
var params = {
"TableName": "Accounts",
"KeySchema": [
{ "AttributeName": "id", KeyType: "HASH" }, // Randomly generated UUID
],
"AttributeDefinitions": [
{ "AttributeName": "id", AttributeType: "S" },
{ "AttributeName": "email", AttributeType: "S" }, // User e-mail.
{ "AttributeName": "password", AttributeType: "S" }, // Hashed password.
],
"GlobalSecondaryIndexes": [
{
"IndexName": "EmailPasswordIndex",
"ProvisionedThroughput": {
"ReadCapacityUnits": 1,
"WriteCapacityUnits": 1
},
"KeySchema": [
{ "AttributeName": "email", KeyType: "HASH" },
{ "AttributeName": "password", KeyType: "RANGE" },
],
"Projection": { "ProjectionType": "ALL" }
},
],
ProvisionedThroughput: {
ReadCapacityUnits: 1,
WriteCapacityUnits: 1
}
};
dynamodb.createTable(params, function(err, data) {
if (err)
console.log(JSON.stringify(err, null, 2));
else
console.log(JSON.stringify(data, null, 2));
});
DynamoDB's Global Secondary Indexes allow for the indexes to be sparse. That means that if you have a GSI whose hash or range key for an item is not defined then that item will simply not be included in the GSI. This is useful in a number of use cases as it allows you to directly identify records that contain certain fields. However, this approach will not work if you are looking for the lack of a field.
To get all of the items that have a field not set your best bet may be resorting to a scan with a filter. This operation will be very expensive but it would be straightforward code looking something like the following:
var params = {
TableName: "Accounts",
FilterExpression: "attribute_not_exists(email)"
};
dynamodb.scan(params, {
if (err)
console.log(JSON.stringify(err, null, 2));
else
console.log(JSON.stringify(data, null, 2));
});
#jaredHatfield is correct if the field does not exist but that will not work if the filed is null. NULL is a keyword and can't used directly. But you can use it with ExpressionAttributeValues.
const params = {
TableName: "Accounts",
FilterExpression: "attribute_not_exists(email) or email = :null",
ExpressionAttributeValues: {
':null': null
}
}
dynamodb.scan(params, (err, data) => {
if (err)
console.log(JSON.stringify(err, null, 2));
else
console.log(JSON.stringify(data, null, 2));
})
Since DynamoDB is what it is, one needs to use non-orthodox approaches to the database usage.
I simply introduced a special value, which may be anything safely recognizable in your domain (e.g. "--NULL--"), and convert that from/to null at the lowest data layer.
Querying entries with that field null is then just querying for that special value.
It's not nice from a perspective of someone used to SQL, but better than scanning.
For the legacy entries, you will need one-time migration.

Resources