Why does a scan with nodejs shows just 3 result, while the dynamodb admin tool shows 9
var params = {
TableName: process.env.DYNAMODB_TABLE_LIGHTHOUSE,
FilterExpression: '#blickArticleId = :valblickArticleId AND #firstFolder = :valfirstFolder',
ExpressionAttributeNames: {
'#blickArticleId': 'blickArticleId',
'#firstFolder': 'firstFolder'
},
ExpressionAttributeValues: {
':valblickArticleId': 'null',
':valfirstFolder': 'null'
},
};
const queryResponse = await dynamoDb.scan(params).promise()
isnt that the same
Are you sure , your scanned content is not more than 1MB ?.
If the total number of scanned items exceeds the maximum data set size
limit of 1 MB, the scan stops and results are returned to the user
with a LastEvaluatedKey
Then you can scan the remaining items by using LastEvaluatedKey.
as jarmod mentioned pagination is the solution:
const getLalalalalal = async () => {
var params = {
TableName: process.env.DYNAMODB_TABLE_LIGHTHOUSE,
FilterExpression: '#blickArticleId = :valblickArticleId AND #firstFolder = :valfirstFolder',
ExpressionAttributeNames: {
'#blickArticleId': 'blickArticleId',
'#firstFolder': 'firstFolder'
},
ExpressionAttributeValues: {
':valblickArticleId': 'null',
':valfirstFolder': 'null'
},
};
return await scanTable(params)
}
const scanTable = async (params) => {
let scanResults = [];
let items;
do {
items = await dynamoDb.scan(params).promise();
items.Items.forEach((item) => scanResults.push(item));
params.ExclusiveStartKey = items.LastEvaluatedKey;
} while(typeof items.LastEvaluatedKey != "undefined");
return scanResults;
};
Your code needs to scan for the remaining items. The presence of a non-null LastEvaluatedKey indicates that DynamoDB results are paginated. The AWS DynamoDB console is presumably doing the pagination for you.
Here is an example of code to fetch all items if paginated.
Related
I've been attempting to scan the table so I can have more functionality with the data. I'm able to .get from the table successfully, but I can't seem to get the scanning function right.
Sample Table:
controlID(N)
controlFunction(S)
1
Protect
2
Assess
3
Protect
Code:
const AWS = require("aws-sdk");
const dynamo = new AWS.DynamoDB.DocumentClient();
exports.handler = async (event, context) => {
let controlInfo;
let body;
let statusCode = 200;
const headers = {
"Content-Type": "application/json"
};
try {
controlInfo = await dynamo
.scan({
FilterExpression: "controlFunction = :cF",
ExpressionAttributeValues: {
":cF": { N: "5" }
},
ProjectionExpression: "controlID",
TableName: "testControls",
})
.promise();
} catch (err) {
statusCode = 400;
controlInfo = err.message;
} finally {
//controlInfo = JSON.stringify(controlInfo);
}
body = {
"Control Info" : controlInfo,
"Threat Info" : "placeHolder"
};
body = JSON.stringify(body);
return {
statusCode,
body,
headers
};
};`
I was expecting the output to be the items of the table with the specified "controlFunction".
Here are the results I get from running the current script:
{
"Control Info": {
"Items": [],
"Count": 0,
"ScannedCount": 115
},
"Threat Info": "placeHolder"
}
You're using the DocumentClient, which auto-marshalls attribute values on both requests to the SDK and responses from the SDK. That means that you don't need to tell the SDK what type each attribute value is. The SDK will automatically map types between DynamoDB native types and their JavaScript equivalents.
So, instead of:
":attrname": { type: value }
You should use:
":attrname": value
For example:
":cF": 5
I have an application written in Lambda functions (with DynamoDB). It performs basic CRUD operations. While adding a new item, I want to check if an item with that same name exists already for the same user.
I implemented the logic below (the POST request). While testing on Postman I get the error "Query condition missed key schema element: user_group_id". I am actually not interested in the user_group_id cause it's only created after the group has been added.
Is there a better or special way of doing this?
What I'm I not doing right?
Thanks
const AWS = require("aws-sdk");
const { v4: uuid } = require("uuid");
const dynamo = new AWS.DynamoDB.DocumentClient();
exports.handler = async (event, context) => {
let body;
let statusCode = 200;
const headers = {
"Content-Type": "application/json",
};
try {
switch (event.routeKey) {
case "GET /user-groups/all/{id}":
const params1 = {
ExpressionAttributeValues: {
":s": parseInt(event.pathParameters.id, 10),
},
KeyConditionExpression: "user_id = :s",
TableName: "User-Groups",
};
body = await dynamo.query(params1).promise();
break;
case "PUT /user-groups/edit":
...
break;
case "PUT /user-groups": //THIS IS THE POST REQUEST
let requestJSON1 = JSON.parse(event.body);
//Check if the name already exists
const name_check = {
ExpressionAttributeValues: {
":u": requestJSON1.user_id,
":g": requestJSON1.user_group_name,
},
KeyConditionExpression: "user_id = :u and user_group_name = :g",
TableName: "User-Groups",
};
const name_exist = await dynamo.query(name_check).promise();
if(name_exist){
throw new Error(`This name already exist`);
}
const arr = requestJSON1.user_ids;
arr.push(requestJSON1.user_id);
const id = uuid();
if(requestJSON1.user_id && requestJSON1.user_group_name && requestJSON1.user_group_tags.length > 0 && requestJSON1.user_ids.length > 1){
body = await dynamo
.put({
TableName: "User-Groups",
Item: {
user_id: requestJSON1.user_id,
user_group_id: id,
user_group_name: requestJSON1.user_group_name,
user_group_tags: requestJSON1.user_group_tags,
user_ids: arr,
created_at: new Date().toGMTString(),
updated_at: new Date().toGMTString(),
},
})
.promise();
body = {
user_id: requestJSON1.user_id,
user_group_id: id,
user_group_name: requestJSON1.user_group_name,
user_group_tags: requestJSON1.user_group_tags,
user_ids: arr,
created_at: new Date().toGMTString(),
updated_at: new Date().toGMTString(),
};
break;
}
break;
case "DELETE /user-groups/{id}/{user_group_id}":
...
break;
default:
throw new Error(`Unsupported route: "${event.routeKey}"`);
}
} catch (err) {
statusCode = 400;
body = err.message;
} finally {
body = JSON.stringify(body);
}
return {
statusCode,
body,
headers,
};
};
ExpressionAttributeValues: {
":u": requestJSON1.user_id,
":g": requestJSON1.user_group_name,
},
KeyConditionExpression: "user_id = :u and user_group_name = :g",
Your issue is here. You set the KeyConditionExpression to state user_id equals :u and user_group_name equals :g However none of those are your tables partition key which is what a Query operation expects.
This means you either need to do a Scan with a FilterExpression or create a GSI with the keys user_id and user_group_name and do your query against your GSI.
I have a collection of lubes. A lube can either be available or out-of-stock. I would like to query max of 5 lubes at a time.
When I'm not using a filter, the continuation token is returned and works fine.
// <Get all lubes / first page (max: 5) >
async getAll(token?: string, limit = 5) {
if (!token) this.store.lube.removeAll();
const {
resources: items,
continuationToken,
hasMoreResults,
} = await this.dbContainer.items
.readAll<ILube>({
maxItemCount: limit,
continuationToken: token,
// partitionKey: "HFO",
})
.fetchNext();
this.store.lube.load(items);
return { hasMoreResults, continuationToken };
}
However, when I want to filter by available lubes, or out-of-stock, using WHERE, the continuation token is returned, however, when I retrieved the data, it doesn't work - no error.
// <Get by availability status>
async getByAvailability(availability: boolean, token?: string, limit = 5) {
if (!token) this.store.lube.removeAll();
this.store.lube.removeAll();
const querySpec = {
query: "SELECT * FROM c WHERE c.isAvailable = #availability",
parameters: [
{
name: "#availability",
value: availability,
},
],
};
const {
resources: items,
continuationToken,
hasMoreResults,
} = await this.dbContainer.items
.query<ILube>(querySpec, { maxItemCount: limit })
.fetchNext();
this.store.lube.load(items);
return { hasMoreResults, continuationToken };
}
Bottom line, how can I use continuation token with where?
There's a bug in the code. Continuation token can be used with WHERE.
Mistake 1: I didn't include the continuationToken in the query of getByAvailability
Mistake 2: I removed the items from the store even when the token!=null.
this.store.lube.removeAll();
// <Get by availability status>
async getByAvailability(availability: boolean, token?: string, limit = 5) {
if (!token) this.store.lube.removeAll();
// **Mistake 2**
// this.store.lube.removeAll(); // This line is not needed.
const querySpec = {
query: "SELECT * FROM c WHERE c.isAvailable = #availability",
parameters: [
{
name: "#availability",
value: availability,
},
],
};
**Mistake 1: Query didn't have continuationToken in it, only had the one being returned.**
const {
resources: items,
continuationToken,
hasMoreResults,
} = await this.dbContainer.items
.query<ILube>(querySpec, { maxItemCount: limit, continuationToken: token })
.fetchNext();
this.store.lube.load(items);
return { hasMoreResults, continuationToken };
}
I am working on lambda function and creating a method for AWS-SDK historical metric report using node, js. The method is running successful but in response showing nothing. Have a look at the response.
Here is my code
function getKeyByValue(object, value) {
return Object.keys(object).find(key =>
object[key] === value);
}
exports.handler = async (event) => {
const AWS = require('aws-sdk');
var connect = new AWS.Connect({ apiVersion: '2017-08-08' });
let queueARN = event.queueARN || null;
const connectInstanceId = process.env.instanceID;
let flag =0, nextToken = null;
let queueARNsObject = {}, queueARNsArray=[], queueTypeObject={},listQueuesResult;
console.log('At line 12 entring do while loop....')
do{
console.log('How many times do I stay here???')
let listQueuesParams = {
InstanceId: connectInstanceId, /* required */
QueueTypes: [
"STANDARD",
],
NextToken: nextToken,
};
let listQueuesPromise = connect.listQueues(listQueuesParams).promise();
listQueuesResult = await listQueuesPromise;
// console.log(listQueuesResult);
listQueuesResult.QueueSummaryList.forEach(queue => {
if(queueARN != null){
if (queue.Arn == queueARN){
queueARNsArray = [queue.Arn];
queueARNsObject[queue.Name]= queue.Arn;
queueTypeObject[queue.QueueType]= queue.Arn;
flag = 1;
return;
}
}else{
queueARNsObject[queue.Name]= queue.Arn;
queueTypeObject[queue.QueueType]= queue.Arn;
queueARNsArray.push(queue.Arn);
nextToken = listQueuesResult.NextToken;
}
});
}while (flag=0 && nextToken != null);
const HistoricalMetrics = [
{
Name : "CONTACTS_HANDLED",
Unit : "COUNT",
Statistic : "SUM"
},
{
Name : "CONTACTS_ABANDONED",
Unit : "COUNT",
Statistic : "SUM"
},
];
// Metrics params
var getHistoricalMetricsParams = {
InstanceId: connectInstanceId,
StartTime: 1593099900,
EndTime: 1593129300,
Filters: {
Channels: ["VOICE"],
Queues: queueARNsArray
},
HistoricalMetrics: HistoricalMetrics,
Groupings: ["QUEUE"]
};
// console.log(getHistoricalMetricsParams);
// get current metrics by queues
var getHistoricalMetricsPromise = connect
.getMetricData(getHistoricalMetricsParams)
.promise();
var getHistoricalMetricsResult = await getHistoricalMetricsPromise;
console.log("historical metrics",getHistoricalMetricsResult);
// console.log("current |||||||| 1 metrics:", JSON.stringify(getCurrentMetricsResult));
let queueMetricsArray = [];
if(getHistoricalMetricsResult.MetricResults.length){
getHistoricalMetricsResult.MetricResults.forEach(queue => {
let queueMetrics = {
"Queue_Name" : getKeyByValue(queueARNsObject ,queue.Dimensions.Queue.Arn),
"CallsHandled": queue.Collections[0].Value,
"CallsAbanoded": queue.Collections[1].Value,
}
queueMetricsArray.push(queueMetrics);
console.log("TYPE||||", getKeyByValue(queueTypeObject ,queue.Dimensions.Queue.Arn))
});
}
const response = {
responseCode: 200,
metricResults: queueMetricsArray
};
return response;
};
I don't have any idea why it is not showing anything. if anyone of you knows please help me to fix it Thanks. I don't know what is Missing I've almost checked everything but I didn't get anything.
There are a few general areas you can look at:
Specify the region.
AWS.Connect({ apiVersion: '2017-08-08', region:'xxxxx' });
use Await directly with listQueues method
let listQueuesPromise = await connect.listQueues(listQueuesParams).promise();
Check Permissions - make sure there is sufficient authority
Lambda Configuration - increase timeout and memory size
PS: What did console log listQueuesPromise return?
I've had a read through AWS's docs around pagination:
As their docs specify:
In a response, DynamoDB returns all the matching results within the scope of the Limit value. For example, if you issue a Query or a Scan request with a Limit value of 6 and without a filter expression, DynamoDB returns the first six items in the table that match the specified key conditions in the request (or just the first six items in the case of a Scan with no filter)
Which means that given I have a table called Questions with an attribute called difficulty(that can take any numeric value ranging from 0 to 2) I might end up with the following conundrum:
A client makes a request, think GET /questions?difficulty=0&limit=3
I forward that 3 to the DynamoDB query, which might return 0 items as the first 3 in the collection might not be of difficulty == 0
I then have to perform a new query to fetch more questions that match that criteria without knowing I might return duplicates
How can I then paginate based on a query correctly? Something where I'll get as many results as I asked for whilst having the correct offset
Using async/await.
const getAllData = async (params) => {
console.log("Querying Table");
let data = await docClient.query(params).promise();
if(data['Items'].length > 0) {
allData = [...allData, ...data['Items']];
}
if (data.LastEvaluatedKey) {
params.ExclusiveStartKey = data.LastEvaluatedKey;
return await getAllData(params);
} else {
return data;
}
}
I am using a global variable allData to collect all the data.
Calling this function is enclosed within a try-catch
try {
await getAllData(params);
console.log("Processing Completed");
// console.log(allData);
} catch(error) {
console.log(error);
}
I am using this from within a Lambda and it works fine.
The article here really helped and guided me. Thanks.
Here is an example of how to iterate over a paginated result set from
a DynamoDB scan (can be easily adapted for query as well) in Node.js.
You could save the LastEvaluatedKey state serverside and pass an identifier back to your client, which it would send with its next request and your server would pass that value as ExclusiveStartKey in the next request to DynamoDB.
const AWS = require('aws-sdk');
AWS.config.logger = console;
const dynamodb = new AWS.DynamoDB({ apiVersion: '2012-08-10' });
let val = 'some value';
let params = {
TableName: "MyTable",
ExpressionAttributeValues: {
':val': {
S: val,
},
},
Limit: 1000,
FilterExpression: 'MyAttribute = :val',
// ExclusiveStartKey: thisUsersScans[someRequestParamScanID]
};
dynamodb.scan(scanParams, function scanUntilDone(err, data) {
if (err) {
console.log(err, err.stack);
} else {
// do something with data
if (data.LastEvaluatedKey) {
params.ExclusiveStartKey = data.LastEvaluatedKey;
dynamodb.scan(params, scanUntilDone);
} else {
// all results scanned. done!
someCallback();
}
}
});
Avoid using recursion to prevent call stack overflow. An iterative solution extending #Roshan Khandelwal's approach:
const getAllData = async (params) => {
const _getAllData = async (params, startKey) => {
if (startKey) {
params.ExclusiveStartKey = startKey
}
return this.documentClient.query(params).promise()
}
let lastEvaluatedKey = null
let rows = []
do {
const result = await _getAllData(params, lastEvaluatedKey)
rows = rows.concat(result.Items)
lastEvaluatedKey = result.LastEvaluatedKey
} while (lastEvaluatedKey)
return rows
}
I hope you figured out. So just in case others might find it useful. AWS has QueryPaginator/ScanPaginator as simple as below:
const paginator = new QueryPaginator(dynamoDb, queryInput);
for await (const page of paginator) {
// do something with the first page of results
break
}
See more details at https://github.com/awslabs/dynamodb-data-mapper-js/tree/master/packages/dynamodb-query-iterator
2022-05-19:
For AWS SDK v3 see how to use paginateXXXX at this blog post https://aws.amazon.com/blogs/developer/pagination-using-async-iterators-in-modular-aws-sdk-for-javascript/
Query and Scan operations return LastEvaluatedKey in their responses. Absent concurrent insertions, you will not miss items nor will you encounter items multiple times, as long as you iterate calls to Query/Scan and set ExclusiveStartKey to the LastEvaluatedKey of the previous call.
For create pagination in dynamodb scan like
var params = {
"TableName" : "abcd",
"FilterExpression" : "#someexperssion=:someexperssion",
"ExpressionAttributeNames" : {"#someexperssion":"someexperssion"},
"ExpressionAttributeValues" : {":someexperssion" : "value"},
"Limit" : 20,
"ExclusiveStartKey" : {"id": "9ee10f6e-ce6d-4820-9fcd-cabb0d93e8da"}
};
DB.scan(params).promise();
where ExclusiveStartKey is LastEvaluatedKey return by this query last execution time
Using async/await, returning the data in await.
Elaboration on #Roshan Khandelwal's answer.
const getAllData = async (params, allData = []) => {
const data = await dynamodbDocClient.scan(params).promise()
if (data['Items'].length > 0) {
allData = [...allData, ...data['Items']]
}
if (data.LastEvaluatedKey) {
params.ExclusiveStartKey = data.LastEvaluatedKey
return await getAllData(params, allData)
} else {
return allData
}
}
Call inside a try/catch:
try {
const data = await getAllData(params);
console.log("my data: ", data);
} catch(error) {
console.log(error);
}
you can do a index secundary by difficulty and at query set KeyConditionExpression where difficulty = 0. Like this
var params = {
TableName: questions,
IndexName: 'difficulty-index',
KeyConditionExpression: 'difficulty = :difficulty ',
ExpressionAttributeValues: {':difficulty':0}
}
You can also achieve this using recrusion instead of a global variable, like:
const getAllData = async (params, allData = []) => {
let data = await db.scan(params).promise();
return (data.LastEvaluatedKey) ?
getAllData({...params, ExclusiveStartKey: data.LastEvaluatedKey}, [...allData, ...data['Items']]) :
[...allData, ...data['Items']];
};
Then you can simply call it like:
let test = await getAllData({ "TableName": "test-table"}); // feel free to add try/catch
Using DynamoDB pagination with async generators:
let items = []
let params = {
TableName: 'mytable',
Limit: 1000,
KeyConditionExpression: 'mykey = :key',
ExpressionAttributeValues: {
':key': { S: 'myvalue' },
},
}
async function* fetchData({
params
}) {
let data
do {
data = await dynamodb.query(params).promise()
yield data.Items
params.ExclusiveStartKey = data.LastEvaluatedKey
} while (typeof data.LastEvaluatedKey != 'undefined')
}
for await (const data of fetchData(params)) {
items = [...items, ...data]
}