Confused on querying DynamoDB - node.js

I've the below data in my DynamoDB.
and I'm trying to achieve the below result.
scan through the table and get the rows where the management is NULL and Location is Midwest.
I initially tried the below query to match the Null.
var scanningParameters = {
TableName: 'LOB',
FilterExpression: "#mgmt contains NULL",
ExpressionAttributeNames: {
"#mgmt": "Management",
}
};
docClient.scan(scanningParameters, onScan);
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
} else {
// print all the movies
console.log("Scan succeeded.");
data.Items.forEach(function (data) {
console.log(
data.lineofbusiness + " name : ",
data.name);
});
if (typeof data.LastEvaluatedKey != "undefined") {
console.log("Scanning for more...");
scanningParameters.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.scan(scanningParameters, onScan);
}
}
}
I get the exception as
{
"message": "Invalid FilterExpression: Syntax error; token: \"contains\", near: \"#mgmtcontains NULL\"",
"code": "ValidationException",
"time": "2017-05-03T13:21:11.611Z",
"requestId": "0T0GU59HRJ24P96D42H9QNC97RVV4KQNSO5AEMVJF66Q9ASUAAJG",
"statusCode": 400,
"retryable": false,
"retryDelay": 13.73953651636839
}
please let me know where am I going wrong and how can I fix this.

Here is the scan item for "NULL" value (i.e. String attribute having NULL as data).
I assume that Management attribute is of String data type containing the string value "NULL".
Code:-
var AWS = require("aws-sdk");
var creds = new AWS.Credentials('akid', 'secret', 'session');
AWS.config.update({
region: "us-west-2",
endpoint: "http://localhost:8000",
credentials : creds
});
var docClient = new AWS.DynamoDB.DocumentClient();
var params = {
TableName: "lob",
FilterExpression: "#mgmt = :mgmtVal",
ExpressionAttributeNames: {
"#mgmt": "Management",
},
ExpressionAttributeValues : {
":mgmtVal" : "NULL"
}
};
docClient.scan(params, onScan);
var count = 0;
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
} else {
console.log("Scan succeeded.");
data.Items.forEach(function(itemData) {
console.log("Item :", ++count,JSON.stringify(itemData));
});
if (typeof data.LastEvaluatedKey != "undefined") {
params.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.scan(params, onScan);
}
}
}

Related

unable to get all items using query DynamoDB

How to scan all items from AWS dynamodb using node.js. I am posting my code here.
//Load the AWS SDK for Node.js
var AWS = require('aws-sdk');
var unmarshalItem = require('dynamodb-marshaler').unmarshalItem;
// Set the region
AWS.config.update({region: 'us-east-1'});
// Create DynamoDB service object
var b = new AWS.DynamoDB({apiVersion: '2012-08-10'});
exports.handler = (event, context, callback) => {
var params = {
TableName: 'IoTdata2',
FilterExpression: "#deviceid = :unitID and #devicetimestamp BETWEEN :ftimestamp and :ttimestamp",
ExpressionAttributeNames: {
"#deviceid": "id",
"#devicetimestamp": "timestamp"
},
ExpressionAttributeValues: {
':unitID': {S: 'arena-MXHGMYzBBP5F6jztnLUdCL' },
':ftimestamp' : {S: '1584022680000' },
':ttimestamp' : {S: '1584023280000' }
},
};
b.scan(params, onScan);
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
} else {
console.log("Scan succeeded.");
var items = data.Items.map(function(val){
return unmarshalItem(val);
})
// continue scanning if we have more items
if (typeof data.LastEvaluatedKey != "undefined") {
console.log("Scanning for more...");
params.ExclusiveStartKey = data.LastEvaluatedKey;
b.scan(params, onScan);
}
}
callback(null, items);
}
};
I have followed the link https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GettingStarted.NodeJs.04.html
I am getting time out here after a while. I have checked this link too
How to fetch/scan all items from `AWS dynamodb` using node.js
I am unable to return data properly i guess, Any suggestions ? Thanks
This worked for me by following Hank solution from here: How to fetch/scan all items from `AWS dynamodb` using node.js
exports.handler = async (event, context, callback) => {
let params = {
ExpressionAttributeNames: {
"#deviceid": "id",
"#devicetimestamp": "timestamp"
},
ExpressionAttributeValues: {
':unitID': {S: event.deviceid },
':ftimestamp' : {S: event.fromtime },
':ttimestamp' : {S: event.totime }
},
KeyConditionExpression: '#deviceid = :unitID and #devicetimestamp BETWEEN :ftimestamp and :ttimestamp',
TableName: 'IoTdata2'
};
let scanResults = [];
let items;
do {
items = await b.query(params).promise();
items.Items.map((item) => scanResults.push(unmarshalItem(item)));
params.ExclusiveStartKey = items.LastEvaluatedKey;
} while (typeof items.LastEvaluatedKey != "undefined");
var len = scanResults.length;
console.log(len);
callback(null, scanResults);
};

Dynamodb alexa query not calling data item

I am creating a skill where I want to call alexa to read differnt items at certain dates and times.
I currently have my table set up as follows:
Date|Time|State
Date is setup as my primary key with time as my sort key. I have setup date and time as slot values in ASK, and I can see that these values are being passed through. I have also made sure the format of my date and time is correct within dynamodb.
My issue is when I call alexa and ask for the state at a certain date and time, I can't get alexa to respond with the state corresponding with that date and time.
Can anyone help me with this? Or tell me where i'm going wrong, I will insert my code below.
const awsSDK = require('aws-sdk');
const updatedincident = 'updatedincident';
const docClient = new awsSDK.DynamoDB.DocumentClient();
var AWSregion = 'us-east-1'; // us-east-1
var AWS = require('aws-sdk');
var dbClient = new AWS.DynamoDB.DocumentClient();
AWS.config.update({
region: "'us-east-1'"
});
let GetMachineStateIntent = (context, callback, dateSlot, timeSlot) => {
var params = {
TableName: "updatedincident",
KeyConditionExpression: 'date = :dVal and time < :tVal',
ExpressionAttributeValues: {
':dVal': dateSlot,
':tVal': timeSlot
},
ScanIndexForward: false // gets values in reverse order by time
};
dbClient.query(params, function (err, data) {
if (err) {
// failed to read from table for some reason..
console.log('failed to load data item:\n' + JSON.stringify(err, null, 2));
// let skill tell the user that it couldn't find the data
sendResponse(context, callback, {
output: "the data could not be loaded from your database",
endSession: false
});
} else {
let dataItem = data.Items[0];
console.log('loaded data item:\n' + JSON.stringify(dataItem, null, 2));
// assuming the item has an attribute called "state"..
sendResponse(context, callback, {
output: dataItem.state,
endSession: false
});
}
});
};
function sendResponse(context, callback, responseOptions) {
if(typeof callback === 'undefined') {
context.succeed(buildResponse(responseOptions));
} else {
callback(null, buildResponse(responseOptions));
}
}
function buildResponse(options) {
var alexaResponse = {
version: "1.0",
response: {
outputSpeech: {
"type": "SSML",
"ssml": `<speak><prosody rate="slow">${options.output}</prosody></speak>`
},
shouldEndSession: options.endSession
}
};
if (options.repromptText) {
alexaResponse.response.reprompt = {
outputSpeech: {
"type": "SSML",
"ssml": `<speak><prosody rate="slow">${options.reprompt}</prosody></speak>`
}
};
}
return alexaResponse;
}
exports.handler = (event, context, callback) => {
try {
var request = event.request;
if (request.type === "LaunchRequest") {
sendResponse(context, callback, {
output: "welcome to my skill, I can tell you about the status of machines at different times. what data are you looking for?",
endSession: false
});
} else if (request.type === "IntentRequest") {
if (request.intent.name === "GetMachineStateIntent") {
var dateSlot = request.intent.slots.Date != null
? request.intent.slots.Date. value : null;
var timeSlot = request.intent.slots.Time != null
? request.intent.slots.Time.value : null;
// pass the slot values to the GetMachineStateIntent function
GetMachineStateIntent(context, callback, dateSlot, timeSlot);
} else if (request.intent.name === "AMAZON.StopIntent" || request.intent.name === "AMAZON.CancelIntent") {
sendResponse(context, callback, {
output: "ok. good bye!",
endSession: true
});
}
else if (request.intent.name === "AMAZON.HelpIntent") {
sendResponse(context, callback, {
output: "you can ask me about incidents that have happened or states of machines in the past",
reprompt: "what can I help you with?",
endSession: false
});
}
else {
sendResponse(context, callback, {
output: "I don't know that one! please try again!",
endSession: false
});
}
}
else if (request.type === "SessionEndedRequest") {
sendResponse(context, callback, ""); // no response needed
}
else {
// an unexpected request type received.. just say I don't know..
sendResponse(context, callback, {
output: "I don't know that one! please try again!",
endSession: false
});
}
} catch (e) {
// handle the error by logging it and sending back an failure
console.log('Unexpected error occurred in the skill handler!', e);
if(typeof callback === 'undefined') {
context.fail("Unexpected error");
} else {
callback("Unexpected error");
}
}
};
The response i currently get with my above code is
'the data could not be loaded from your database'
Cloud watch also tells me this
2018-05-16T09:29:06.635Z 93d4b6e6-58eb-11e8-b686-597d65771e90 failed to load data item:
{
"message": "Invalid KeyConditionExpression: Attribute name is a reserved keyword; reserved keyword: date",
"code": "ValidationException",
"time": "2018-05-16T09:29:06.633Z",
"requestId": "EQPQTAGO4QKH9SM5GSOA9O3DDFVV4KQNSO5AEMVJF66Q9ASUAAJG",
"statusCode": 400,
"retryable": false,
"retryDelay": 35.56027710686527
}
In the GetMachineStateIntent function, try changing the params structure like this:
var params = {
TableName: "updatedincident",
KeyConditionExpression: '#d = :dVal and #t < :tVal',
ExpressionAttributeValues: {
':dVal': dateSlot,
':tVal': timeSlot
},
ExpressionAttributeNames: {
'#d': 'date',
'#t': 'time'
},
ScanIndexForward: false // gets values in reverse order by time
};
It looks like the word date is a reserved keyword so it can't be used directly in an expression such as date = :dVal which is why you must give it an attribute name alias (#d) which maps back to the actual attribute name (date).
In DynamoDB, you have to use two keys i.e Primary key and Primary sort key. The query searches on the basis of these two keys for the requested value.
Try my code:
'FindName': function() {
var tableName = "CVRaman";
var userName = "Prateek";
var userId = "kapoor";
const dynamodbParams = {
TableName: tableName,
Key: {
userId: userId,
userName: userName,
},
ProjectionExpression: 'userName', //Projection Expression is used to select only specific columns which we want to get
};
console.log("Attempting to find name", dynamodbParams);
dynamoDb.get(dynamodbParams).promise()
.then(data => {
console.log('name found', dynamodbParams);
console.log(data.Item.userName);
var a = data.Item.userName;
console.log(a);
this.emit(':ask', 'Name as ' + a);
})
.catch(err => {
console.log(err);
this.emit(':tell', 'we have a problem');
});
},

How to scan in dynamoDB based on the values in an array?

here is my code where i have a list of values in an array. I need to fetch all the projects from project table which matches the id's in the array.
var arr=[];
for(var index in data.Items){
if(data.Items[index].hasOwnProperty('projectId'))
arr.push(data.Items[index].projectId);
};
var params = {
TableName: 'projects',
FilterExpression: 'id IN (:id)',
ExpressionAttributeValues: {
':id': arr
}
};
dynamodbclient.scan(params, function (err, docs) {
if (err) {
console.log("Error", err);
} else {
console.log("Success");
callback(err, docs.Items);
}
});
However i am not getting the proper results.
Option 1 - Static :-
If you know all the values its count upfront, please construct the FilterExpression as mentioned below:-
var params = {
TableName : "projects",
FilterExpression : "id IN (:id1, :id2)",
ExpressionAttributeValues : {
":id1" : "id val 1",
":id2" : "id val 2"
}
};
Option 2 - Dynamic:-
var titleValues = ["The Big New Movie 2012", "The Big New Movie"];
var titleObject = {};
var index = 0;
titleValues.forEach(function(value) {
index++;
var titleKey = ":titlevalue"+index;
titleObject[titleKey.toString()] = value;
});
var params = {
TableName : "Movies",
FilterExpression : "title IN ("+Object.keys(titleObject).toString()+ ")",
ExpressionAttributeValues : titleObject
};
docClient.scan(params, onScan);
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(
err, null, 2));
} else {
// print all the movies
console.log("Scan succeeded.");
data.Items.forEach(function(movie) {
console.log("Item :", JSON.stringify(movie));
});
// continue scanning if we have more movies
if (typeof data.LastEvaluatedKey != "undefined") {
console.log("Scanning for more...");
params.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.scan(params, onScan);
}
}
}

Aws lambda async and sync issue

I have a requirement where i need to query on a table then loop through the result and query another table inside a loop .. But i tried using synchronize node js module . But sometime api ends with timeout error with this method. Now i am trying my code in below way but no luck.
var notification = {
TableName: "Tablename",
KeyConditionExpression:"#userId =:userid and #datetime <=:datetime",
ScanIndexForward: false,
ExpressionAttributeNames: {
"#userId": "userId",
"#datetime":"datetime"
},
ExpressionAttributeValues: {
":userid" :userId,
":datetime" :datetime
}
};
docClient.query(notification, function(err, result) {
if (err) {
context.succeed({success: false, message : JSON.stringify(err, null, 2),method:"notification",type:type});
}else{
result = result.Items;
if (result.length>0) {
for(var i=0;i<result.length;i++){
result[k]['CafeDetail']=getDetail(result[k].CafeID,function (data) {
});
if (k ==(result.length -1)) {
context.succeed({success: true,data:result, message : "Notification list",method:"notification",type:type});
}
}
}
}
});
function getDetail(CafeID,callback) {
var param = {
TableName: "Tablename",
ProjectionExpression:"CafeName,Cafe_MainImage",
KeyConditionExpression:"CafeID =:cafeID",
ExpressionAttributeValues: {
":cafeID" :CafeID
}
};
docClient.query(param, function(err, CafeDetail) {
if (err) {
console.log (err)
callback(err);
} else {
callback(CafeDetail.Items);
console.log(CafeDetail.Items);
}
});
}
On output this variable (result[k]['CafeDetail']) is always coming as undefined . However console.log inside getDetail function print the result.
I am newbie to aws lambda with node js .. Any help will be appreciated .
Thanks in Advance
I just got solution to my question . So i am posting it,may be it will help someone else facing same issue
var notification = {
TableName: "Tablename",
KeyConditionExpression:"#userId =:userid",
Limit: 12,
ScanIndexForward: false,
ExpressionAttributeNames: {
"#userId": "userId",
},
ExpressionAttributeValues: {
":userid" :userId
}
};
docClient.query(notification, function(err, result) {
if (err) {
context.succeed({success: false, message : JSON.stringify(err, null, 2),method:"notification",type:type});
}else{
var ret =[];
var results =result.Items;
if (results.length>0) {
results.forEach(function(result, i) {
console.log(i);
var param = {
TableName: "Tablename",
ProjectionExpression:"CafeName,Cafe_MainImage",
KeyConditionExpression:"CafeID =:cafeID",
ExpressionAttributeValues: {
":cafeID" :result.CafeID
}
};
docClient.query(param, function(err, CafeDetail) {
if (err) {
context.succeed({success: false, message : JSON.stringify(err, null, 2),method:"notification",type:type});
} else {
console.log(CafeDetail.Items);
CafeDetail =CafeDetail.Items;
result['CafeDetail']=CafeDetail
console.log(result);
if (i === results.length -1) { // last iteration
context.succeed({success: true,data:results, message : "Notification list",method:"notification",type:type});
}
}
});
});
}else{
context.succeed({success: true,data:result, message : "Notification list",method:"notification",type:type});
}
}
});

How to reformat DynamoDB response from nodejs client

When I make a get request from dynamoDB using the aws nodejs client the result contains a response with the type as the Key i.e.
{
Item: {
newrelic_id: {
S: 'nr-1234'
}
,
jumpcloud_id: {
S: 'j-234'
}
,
microtime: {
N: '1475690490854'
}
,
instance_id: {
S: 'i-abc1234'
}
}
}
Note that the keys to the values are the prefixes for the types , S for String and N for Number is there a way to remove this "type key"?
Here is the sample code using DocumentClient.
var AWS = require("aws-sdk");
var creds = new AWS.Credentials('akid', 'secret', 'session');
AWS.config.update({
region : "us-west-2",
endpoint : "http://localhost:8000",
credentials : creds
});
var docClient = new AWS.DynamoDB.DocumentClient();
var table = "Movies";
var year_val = 2015;
var title = "The Big New Movie";
var params = {
TableName : table,
KeyConditionExpression : 'yearkey = :hkey and title = :rkey',
ExpressionAttributeValues : {
':hkey' : year_val,
':rkey' : title
}
};
docClient.query(params, function(err, data) {
if (err) {
console.error("Unable to read item. Error JSON:", JSON.stringify(err,
null, 2));
} else {
console.log("GetItem succeeded:", JSON.stringify(data, null, 2));
}
});
Output:-
The output doesn't have the data type of the attribute.
GetItem succeeded: {
"Items": [
{
"title": "The Big New Movie",
"yearkey": 2015,
"info": {
"rating": 0,
"plot": "Nothing happens at all."
}
}
],
"Count": 1,
"ScannedCount": 1
}

Resources