Inconsistent results with query/scan DynamoDB - NodeJS - node.js

Question

I have a DynamoDB table that looks like this:
(there are like 1500000 entries like this one with different timestamps)
I have 2 GSI:
I'm trying to retrieve all the rows in the table for a given day.
This is what my code looks like (NodeJS):
var AWS = require("aws-sdk");
AWS.config.update({accessKeyId: "", secretAccessKey: ""});
AWS.config.update({region: 'us-east-1'});
var docClient = new AWS.DynamoDB.DocumentClient();
var params = {
TableName: "QfGamingTransactionsProd",
IndexName: 'Result-RedeemedAt-index',
KeyConditionExpression: "#rs = :result and begins_with (#rat, :Rat)",
ExpressionAttributeNames: {
"#rs": "Result",
"#rat": "RedeemedAt"
},
ExpressionAttributeValues: {
":result": "SUCCESS",
":Rat": "2016-10-20"
}
};
docClient.query(params, function (err, data) {
if (err) {
console.error("Unable to query. Error:", JSON.stringify(err, null, 2));
} else {
console.log("\nQuery succeeded. \n");
console.log("- Total", data.Count);
}
});
It seems to be working, but i'm getting (way) less results than expected. This same code works fine on a smaller Table.
Similar results with "Scan".
What am I missing?

Answer 1

According to the size of each record, the number of records retrieved will change since DynamoDB has a size limitation for query (1MB).
In DynamoDB, a query will return only 1MB of data.
But we can paginate through the results. It may solve your issue.
Data returned by the query will contain a "LastEvaluatedKey", if data that satisfy that query is not fully retrieved. So we have to set the "LastEvaluatedKey" as the "ExclusiveStartKey". Then the query will retrieve the remaining data. By recursively following this method, we will get the complete data.
var data = [];
async.until(function () {
return scanComplete;
},
function (callback) {
docClient.query(params, function (err, result) {
if (err) {
console.log(err);
} else {
data.push(result.Items);
if (typeof (result.LastEvaluatedKey) === 'undefined') {
scanComplete = true;
//fully retrieved
} else {
params.ExclusiveStartKey = result.LastEvaluatedKey;
}
if (!scanComplete) {
}
}
callback(err);
});
},
// this runs when the loop is complete or returns an error
function (err) {
if (err) {
console.log('error in processing scan ');
console.log(err);
reject(err);
} else {
resolve(data);
}
});

Answer 2

This is because by default DynamoDB will return only 1mb of data at a time, But there is a way to solve this issue.
You need to change your implementation like following
Step 1: Call DyanmoDB table, it will return 1st 1mb of data, with that it will return "Next Evaluated Key"
Step 2: Call Dynamodb table again but this time you pass "Next Evaluated Key" in "Exclusive Start key" to get new set of data
Step3: Check if "Next Evaluated Key" is still available then repeat step2 else you got all the data for that key
Here are the references:
About query limits
Blog on how to implement this code
Hope that helps