I've been attempting to scan the table so I can have more functionality with the data. I'm able to .get from the table successfully, but I can't seem to get the scanning function right.
Sample Table:
controlID(N)
controlFunction(S)
1
Protect
2
Assess
3
Protect
Code:
const AWS = require("aws-sdk");
const dynamo = new AWS.DynamoDB.DocumentClient();
exports.handler = async (event, context) => {
let controlInfo;
let body;
let statusCode = 200;
const headers = {
"Content-Type": "application/json"
};
try {
controlInfo = await dynamo
.scan({
FilterExpression: "controlFunction = :cF",
ExpressionAttributeValues: {
":cF": { N: "5" }
},
ProjectionExpression: "controlID",
TableName: "testControls",
})
.promise();
} catch (err) {
statusCode = 400;
controlInfo = err.message;
} finally {
//controlInfo = JSON.stringify(controlInfo);
}
body = {
"Control Info" : controlInfo,
"Threat Info" : "placeHolder"
};
body = JSON.stringify(body);
return {
statusCode,
body,
headers
};
};`
I was expecting the output to be the items of the table with the specified "controlFunction".
Here are the results I get from running the current script:
{
"Control Info": {
"Items": [],
"Count": 0,
"ScannedCount": 115
},
"Threat Info": "placeHolder"
}
You're using the DocumentClient, which auto-marshalls attribute values on both requests to the SDK and responses from the SDK. That means that you don't need to tell the SDK what type each attribute value is. The SDK will automatically map types between DynamoDB native types and their JavaScript equivalents.
So, instead of:
":attrname": { type: value }
You should use:
":attrname": value
For example:
":cF": 5
Related
I have an application written in Lambda functions (with DynamoDB). It performs basic CRUD operations. While adding a new item, I want to check if an item with that same name exists already for the same user.
I implemented the logic below (the POST request). While testing on Postman I get the error "Query condition missed key schema element: user_group_id". I am actually not interested in the user_group_id cause it's only created after the group has been added.
Is there a better or special way of doing this?
What I'm I not doing right?
Thanks
const AWS = require("aws-sdk");
const { v4: uuid } = require("uuid");
const dynamo = new AWS.DynamoDB.DocumentClient();
exports.handler = async (event, context) => {
let body;
let statusCode = 200;
const headers = {
"Content-Type": "application/json",
};
try {
switch (event.routeKey) {
case "GET /user-groups/all/{id}":
const params1 = {
ExpressionAttributeValues: {
":s": parseInt(event.pathParameters.id, 10),
},
KeyConditionExpression: "user_id = :s",
TableName: "User-Groups",
};
body = await dynamo.query(params1).promise();
break;
case "PUT /user-groups/edit":
...
break;
case "PUT /user-groups": //THIS IS THE POST REQUEST
let requestJSON1 = JSON.parse(event.body);
//Check if the name already exists
const name_check = {
ExpressionAttributeValues: {
":u": requestJSON1.user_id,
":g": requestJSON1.user_group_name,
},
KeyConditionExpression: "user_id = :u and user_group_name = :g",
TableName: "User-Groups",
};
const name_exist = await dynamo.query(name_check).promise();
if(name_exist){
throw new Error(`This name already exist`);
}
const arr = requestJSON1.user_ids;
arr.push(requestJSON1.user_id);
const id = uuid();
if(requestJSON1.user_id && requestJSON1.user_group_name && requestJSON1.user_group_tags.length > 0 && requestJSON1.user_ids.length > 1){
body = await dynamo
.put({
TableName: "User-Groups",
Item: {
user_id: requestJSON1.user_id,
user_group_id: id,
user_group_name: requestJSON1.user_group_name,
user_group_tags: requestJSON1.user_group_tags,
user_ids: arr,
created_at: new Date().toGMTString(),
updated_at: new Date().toGMTString(),
},
})
.promise();
body = {
user_id: requestJSON1.user_id,
user_group_id: id,
user_group_name: requestJSON1.user_group_name,
user_group_tags: requestJSON1.user_group_tags,
user_ids: arr,
created_at: new Date().toGMTString(),
updated_at: new Date().toGMTString(),
};
break;
}
break;
case "DELETE /user-groups/{id}/{user_group_id}":
...
break;
default:
throw new Error(`Unsupported route: "${event.routeKey}"`);
}
} catch (err) {
statusCode = 400;
body = err.message;
} finally {
body = JSON.stringify(body);
}
return {
statusCode,
body,
headers,
};
};
ExpressionAttributeValues: {
":u": requestJSON1.user_id,
":g": requestJSON1.user_group_name,
},
KeyConditionExpression: "user_id = :u and user_group_name = :g",
Your issue is here. You set the KeyConditionExpression to state user_id equals :u and user_group_name equals :g However none of those are your tables partition key which is what a Query operation expects.
This means you either need to do a Scan with a FilterExpression or create a GSI with the keys user_id and user_group_name and do your query against your GSI.
I have a DynamoDB table with columns and primary key as ipAddress:
ipAddress
visits
I am fetching the IP address of the user from my react website and inserting it to DynamoDB via a Lambda function and API Gateway POST request.
If the IP address coming from the React website is already present in the DynamoDB, then increment the value in visits column. If not, then create a new record with the new IP address and visits = 1. I tried using ConditionExpression but to no avail.
So far, I am only inserting the ipAddress using Lambda. Below is my Lambda function in NodeJS:
const AWS = require("aws-sdk");
const documentClient = new AWS.DynamoDB.DocumentClient();
exports.handler = async event => {
const { ipAddress} = JSON.parse(event.body);
const params = {
TableName: "ipAddress",
Item: {
ipAddress: ipAddress,
}
};
try {
const data = await documentClient.put(params).promise();
const response = {
statusCode: 200,
};
return response;
}
catch (e) {
return {
statusCode: 500,
body: JSON.stringify(e)
};
}
};
We need to use update-item rather than put-item, both creates a record if record doesn't exist. But update-item accepts UpdateExpression to help us update an existing attribute.
UpdateExpression: "SET visits = if_not_exists(visits, :initial) + :num"
if_not_exists helps to use initial value for the first time when attributes visits doesn't exist.
docClient.update(
{
TableName: "ipAddress",
Key: {
ipAddress: ipAddress,
},
UpdateExpression: "SET visits = if_not_exists(visits, :initial) + :num",
ExpressionAttributeValues: {
":num": 1,
":initial": 0,
},
},
function (error, result) {
console.log("error", error, "result", result);
}
);
I am working on lambda function and creating a method for AWS-SDK historical metric report using node, js. The method is running successful but in response showing nothing. Have a look at the response.
Here is my code
function getKeyByValue(object, value) {
return Object.keys(object).find(key =>
object[key] === value);
}
exports.handler = async (event) => {
const AWS = require('aws-sdk');
var connect = new AWS.Connect({ apiVersion: '2017-08-08' });
let queueARN = event.queueARN || null;
const connectInstanceId = process.env.instanceID;
let flag =0, nextToken = null;
let queueARNsObject = {}, queueARNsArray=[], queueTypeObject={},listQueuesResult;
console.log('At line 12 entring do while loop....')
do{
console.log('How many times do I stay here???')
let listQueuesParams = {
InstanceId: connectInstanceId, /* required */
QueueTypes: [
"STANDARD",
],
NextToken: nextToken,
};
let listQueuesPromise = connect.listQueues(listQueuesParams).promise();
listQueuesResult = await listQueuesPromise;
// console.log(listQueuesResult);
listQueuesResult.QueueSummaryList.forEach(queue => {
if(queueARN != null){
if (queue.Arn == queueARN){
queueARNsArray = [queue.Arn];
queueARNsObject[queue.Name]= queue.Arn;
queueTypeObject[queue.QueueType]= queue.Arn;
flag = 1;
return;
}
}else{
queueARNsObject[queue.Name]= queue.Arn;
queueTypeObject[queue.QueueType]= queue.Arn;
queueARNsArray.push(queue.Arn);
nextToken = listQueuesResult.NextToken;
}
});
}while (flag=0 && nextToken != null);
const HistoricalMetrics = [
{
Name : "CONTACTS_HANDLED",
Unit : "COUNT",
Statistic : "SUM"
},
{
Name : "CONTACTS_ABANDONED",
Unit : "COUNT",
Statistic : "SUM"
},
];
// Metrics params
var getHistoricalMetricsParams = {
InstanceId: connectInstanceId,
StartTime: 1593099900,
EndTime: 1593129300,
Filters: {
Channels: ["VOICE"],
Queues: queueARNsArray
},
HistoricalMetrics: HistoricalMetrics,
Groupings: ["QUEUE"]
};
// console.log(getHistoricalMetricsParams);
// get current metrics by queues
var getHistoricalMetricsPromise = connect
.getMetricData(getHistoricalMetricsParams)
.promise();
var getHistoricalMetricsResult = await getHistoricalMetricsPromise;
console.log("historical metrics",getHistoricalMetricsResult);
// console.log("current |||||||| 1 metrics:", JSON.stringify(getCurrentMetricsResult));
let queueMetricsArray = [];
if(getHistoricalMetricsResult.MetricResults.length){
getHistoricalMetricsResult.MetricResults.forEach(queue => {
let queueMetrics = {
"Queue_Name" : getKeyByValue(queueARNsObject ,queue.Dimensions.Queue.Arn),
"CallsHandled": queue.Collections[0].Value,
"CallsAbanoded": queue.Collections[1].Value,
}
queueMetricsArray.push(queueMetrics);
console.log("TYPE||||", getKeyByValue(queueTypeObject ,queue.Dimensions.Queue.Arn))
});
}
const response = {
responseCode: 200,
metricResults: queueMetricsArray
};
return response;
};
I don't have any idea why it is not showing anything. if anyone of you knows please help me to fix it Thanks. I don't know what is Missing I've almost checked everything but I didn't get anything.
There are a few general areas you can look at:
Specify the region.
AWS.Connect({ apiVersion: '2017-08-08', region:'xxxxx' });
use Await directly with listQueues method
let listQueuesPromise = await connect.listQueues(listQueuesParams).promise();
Check Permissions - make sure there is sufficient authority
Lambda Configuration - increase timeout and memory size
PS: What did console log listQueuesPromise return?
I have created an AWS API Gateway to invoke a Lambda function to generate random numbers:
Lambda Function :
exports.handler = (event, context, callback) => {
let min = parseInt(event.min);
let max = parseInt(event.max);
let generatedNumber = Math.floor(Math.random() * max) + min;
context.done(null, {generatedNumber: generatedNumber});
};
Body mapping Template in API gateway for get method:
{
"min" : $input.params('min'),
"max" : $input.params('max')
}
When I access API endpoint like below:
https://abcdefgh.execute-api.ap-south-1.amazonaws.com/DEV/number?min=10&max=20
I get the proper response :
{"generatedNumber":28}
But when I try to access the API in node.js using aws-api-gateway-client I am receiving the below response :
_currentUrl: 'https://abcdefgh.execute-api.ap-south-1.amazonaws.com/DEV/number' },
response: undefined
The current url should be set to 'https://abcdefgh.execute-api.ap-south-1.amazonaws.com/DEV/number?min=20&max=40' but it is set to 'https://abcdefgh.execute-api.ap-south-1.amazonaws.com/DEV/number'.
Here is my node.js code to access this api:
let AWS = require('aws-sdk');
AWS.config.loadFromPath('./config.json');
//AWS.config.region = 'ap-south-1';
let lambda = new AWS.Lambda();
let apigClientFactory = require('aws-api-gateway-client').default;
let config = {
invokeUrl: 'https://abcdefgh.execute-api.ap-south-1.amazonaws.com/DEV',
accessKey: '<access-key>',
secretKey: '<secret-key>',
region: 'ap-south-1'
};
let apigClient = apigClientFactory.newClient(config);
let apiParams = '{"min": 20,"max": 40}';
let body = {
}
let additionalParams = {
}
apigClient.invokeApi(apiParams, '/number', 'GET', additionalParams, body)
.then(function (result) {
console.log(result);
})
.catch(function (error) {
console.log(error);
});
I tried changing apiParams to :
let apiParams = {"min": 20,"max": 40};
The I receive the below error:
'{"message": "Could not parse request body into json: Unexpected character (\\\',\\\' (code 44)): expected a value\\n at [Source: [B#42feb146; line: 2, column: 14]"}' } }
What is wrong in my code?
Thanks in advance
Try modifying the mapping template:
{
"min" : "$input.params('min')",
"max" : "$input.params('max')"
}
Source: input-variable-reference
I found the problem. I need to pass parameters in additionalParmaeters object like :
let additionalParams = {
queryParams: {
min: 20, max: 40
}
}
But the text
var params = {
//This is where any header, path, or querystring request params go. The key is the parameter named as defined in the API
userId: '1234',
};
is misleading because query parameters were not passed when parameters were added to params object ( maybe it was for me ), but were only passed when passed inside additionalPrams.
Hope it helps.
I've had a read through AWS's docs around pagination:
As their docs specify:
In a response, DynamoDB returns all the matching results within the scope of the Limit value. For example, if you issue a Query or a Scan request with a Limit value of 6 and without a filter expression, DynamoDB returns the first six items in the table that match the specified key conditions in the request (or just the first six items in the case of a Scan with no filter)
Which means that given I have a table called Questions with an attribute called difficulty(that can take any numeric value ranging from 0 to 2) I might end up with the following conundrum:
A client makes a request, think GET /questions?difficulty=0&limit=3
I forward that 3 to the DynamoDB query, which might return 0 items as the first 3 in the collection might not be of difficulty == 0
I then have to perform a new query to fetch more questions that match that criteria without knowing I might return duplicates
How can I then paginate based on a query correctly? Something where I'll get as many results as I asked for whilst having the correct offset
Using async/await.
const getAllData = async (params) => {
console.log("Querying Table");
let data = await docClient.query(params).promise();
if(data['Items'].length > 0) {
allData = [...allData, ...data['Items']];
}
if (data.LastEvaluatedKey) {
params.ExclusiveStartKey = data.LastEvaluatedKey;
return await getAllData(params);
} else {
return data;
}
}
I am using a global variable allData to collect all the data.
Calling this function is enclosed within a try-catch
try {
await getAllData(params);
console.log("Processing Completed");
// console.log(allData);
} catch(error) {
console.log(error);
}
I am using this from within a Lambda and it works fine.
The article here really helped and guided me. Thanks.
Here is an example of how to iterate over a paginated result set from
a DynamoDB scan (can be easily adapted for query as well) in Node.js.
You could save the LastEvaluatedKey state serverside and pass an identifier back to your client, which it would send with its next request and your server would pass that value as ExclusiveStartKey in the next request to DynamoDB.
const AWS = require('aws-sdk');
AWS.config.logger = console;
const dynamodb = new AWS.DynamoDB({ apiVersion: '2012-08-10' });
let val = 'some value';
let params = {
TableName: "MyTable",
ExpressionAttributeValues: {
':val': {
S: val,
},
},
Limit: 1000,
FilterExpression: 'MyAttribute = :val',
// ExclusiveStartKey: thisUsersScans[someRequestParamScanID]
};
dynamodb.scan(scanParams, function scanUntilDone(err, data) {
if (err) {
console.log(err, err.stack);
} else {
// do something with data
if (data.LastEvaluatedKey) {
params.ExclusiveStartKey = data.LastEvaluatedKey;
dynamodb.scan(params, scanUntilDone);
} else {
// all results scanned. done!
someCallback();
}
}
});
Avoid using recursion to prevent call stack overflow. An iterative solution extending #Roshan Khandelwal's approach:
const getAllData = async (params) => {
const _getAllData = async (params, startKey) => {
if (startKey) {
params.ExclusiveStartKey = startKey
}
return this.documentClient.query(params).promise()
}
let lastEvaluatedKey = null
let rows = []
do {
const result = await _getAllData(params, lastEvaluatedKey)
rows = rows.concat(result.Items)
lastEvaluatedKey = result.LastEvaluatedKey
} while (lastEvaluatedKey)
return rows
}
I hope you figured out. So just in case others might find it useful. AWS has QueryPaginator/ScanPaginator as simple as below:
const paginator = new QueryPaginator(dynamoDb, queryInput);
for await (const page of paginator) {
// do something with the first page of results
break
}
See more details at https://github.com/awslabs/dynamodb-data-mapper-js/tree/master/packages/dynamodb-query-iterator
2022-05-19:
For AWS SDK v3 see how to use paginateXXXX at this blog post https://aws.amazon.com/blogs/developer/pagination-using-async-iterators-in-modular-aws-sdk-for-javascript/
Query and Scan operations return LastEvaluatedKey in their responses. Absent concurrent insertions, you will not miss items nor will you encounter items multiple times, as long as you iterate calls to Query/Scan and set ExclusiveStartKey to the LastEvaluatedKey of the previous call.
For create pagination in dynamodb scan like
var params = {
"TableName" : "abcd",
"FilterExpression" : "#someexperssion=:someexperssion",
"ExpressionAttributeNames" : {"#someexperssion":"someexperssion"},
"ExpressionAttributeValues" : {":someexperssion" : "value"},
"Limit" : 20,
"ExclusiveStartKey" : {"id": "9ee10f6e-ce6d-4820-9fcd-cabb0d93e8da"}
};
DB.scan(params).promise();
where ExclusiveStartKey is LastEvaluatedKey return by this query last execution time
Using async/await, returning the data in await.
Elaboration on #Roshan Khandelwal's answer.
const getAllData = async (params, allData = []) => {
const data = await dynamodbDocClient.scan(params).promise()
if (data['Items'].length > 0) {
allData = [...allData, ...data['Items']]
}
if (data.LastEvaluatedKey) {
params.ExclusiveStartKey = data.LastEvaluatedKey
return await getAllData(params, allData)
} else {
return allData
}
}
Call inside a try/catch:
try {
const data = await getAllData(params);
console.log("my data: ", data);
} catch(error) {
console.log(error);
}
you can do a index secundary by difficulty and at query set KeyConditionExpression where difficulty = 0. Like this
var params = {
TableName: questions,
IndexName: 'difficulty-index',
KeyConditionExpression: 'difficulty = :difficulty ',
ExpressionAttributeValues: {':difficulty':0}
}
You can also achieve this using recrusion instead of a global variable, like:
const getAllData = async (params, allData = []) => {
let data = await db.scan(params).promise();
return (data.LastEvaluatedKey) ?
getAllData({...params, ExclusiveStartKey: data.LastEvaluatedKey}, [...allData, ...data['Items']]) :
[...allData, ...data['Items']];
};
Then you can simply call it like:
let test = await getAllData({ "TableName": "test-table"}); // feel free to add try/catch
Using DynamoDB pagination with async generators:
let items = []
let params = {
TableName: 'mytable',
Limit: 1000,
KeyConditionExpression: 'mykey = :key',
ExpressionAttributeValues: {
':key': { S: 'myvalue' },
},
}
async function* fetchData({
params
}) {
let data
do {
data = await dynamodb.query(params).promise()
yield data.Items
params.ExclusiveStartKey = data.LastEvaluatedKey
} while (typeof data.LastEvaluatedKey != 'undefined')
}
for await (const data of fetchData(params)) {
items = [...items, ...data]
}