How to scan all items from AWS dynamodb using node.js. I am posting my code here.
//Load the AWS SDK for Node.js
var AWS = require('aws-sdk');
var unmarshalItem = require('dynamodb-marshaler').unmarshalItem;
// Set the region
AWS.config.update({region: 'us-east-1'});
// Create DynamoDB service object
var b = new AWS.DynamoDB({apiVersion: '2012-08-10'});
exports.handler = (event, context, callback) => {
var params = {
TableName: 'IoTdata2',
FilterExpression: "#deviceid = :unitID and #devicetimestamp BETWEEN :ftimestamp and :ttimestamp",
ExpressionAttributeNames: {
"#deviceid": "id",
"#devicetimestamp": "timestamp"
},
ExpressionAttributeValues: {
':unitID': {S: 'arena-MXHGMYzBBP5F6jztnLUdCL' },
':ftimestamp' : {S: '1584022680000' },
':ttimestamp' : {S: '1584023280000' }
},
};
b.scan(params, onScan);
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
} else {
console.log("Scan succeeded.");
var items = data.Items.map(function(val){
return unmarshalItem(val);
})
// continue scanning if we have more items
if (typeof data.LastEvaluatedKey != "undefined") {
console.log("Scanning for more...");
params.ExclusiveStartKey = data.LastEvaluatedKey;
b.scan(params, onScan);
}
}
callback(null, items);
}
};
I have followed the link https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GettingStarted.NodeJs.04.html
I am getting time out here after a while. I have checked this link too
How to fetch/scan all items from `AWS dynamodb` using node.js
I am unable to return data properly i guess, Any suggestions ? Thanks
This worked for me by following Hank solution from here: How to fetch/scan all items from `AWS dynamodb` using node.js
exports.handler = async (event, context, callback) => {
let params = {
ExpressionAttributeNames: {
"#deviceid": "id",
"#devicetimestamp": "timestamp"
},
ExpressionAttributeValues: {
':unitID': {S: event.deviceid },
':ftimestamp' : {S: event.fromtime },
':ttimestamp' : {S: event.totime }
},
KeyConditionExpression: '#deviceid = :unitID and #devicetimestamp BETWEEN :ftimestamp and :ttimestamp',
TableName: 'IoTdata2'
};
let scanResults = [];
let items;
do {
items = await b.query(params).promise();
items.Items.map((item) => scanResults.push(unmarshalItem(item)));
params.ExclusiveStartKey = items.LastEvaluatedKey;
} while (typeof items.LastEvaluatedKey != "undefined");
var len = scanResults.length;
console.log(len);
callback(null, scanResults);
};
Related
Lambda provides the expected result only when I pass the value manually (Id = '011010').
In the step function the value "ID" value will be random based on the logic from previous step, since the Id value is not static how to handle this scenario "ExpressionAttributeValues"
I tried all the below syntax but no luck..
ExpressionAttributeValues: { ':value': $.External.Id}
ExpressionAttributeValues: { ':value': External.Id.$}
ExpressionAttributeValues: { ':value': $.Id}
ExpressionAttributeValues: { ':value': Id.$}
ExpressionAttributeValues: { ':value': event.Id}
Lambda-Code
'use strict'
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient();
var params = {
TableName: 'temptable',
IndexName: 'Id-CurrentStatus-index',
KeyConditionExpression: '#Id= :value',
ExpressionAttributeNames: { '#Id': 'Id'},
ExpressionAttributeValues: { ':value': 'M1' }
};
async function queryItems(){
try {
const data = await docClient.query(params).promise()
return data
} catch (err) {
return err
}
}
exports.handler = async (event, context) => {
try {
const data = await queryItems()
return { body: JSON.stringify(data) }
} catch (err) {
return { error: err }
}
}
I can read it from the
console.log("Memberid :" + JSON.stringify(event.Id, null, 2))
but how to pass the same value in the
ExpressionAttributeValues: { ':value': 'M1' }
I tried the below syntax. nothing works
ExpressionAttributeValues: { ':value': JSON.stringify(event.Id, null, 2) }
ExpressionAttributeValues: { ':value': event.Id}
ExpressionAttributeValues: { ':value': event.Id}
ExpressionAttributeValues: { ':value': Id}
I got the required code after checking multiple thread.
AWS Lambda function to scan/query DynamoDB table using array values as FilterExpression
'use strict'
var AWS = require('aws-sdk');
var mydocumentClient = new AWS.DynamoDB.DocumentClient();
exports.handler = function (event, context, callback) {
var params = {
TableName: 'temptable',
KeyConditionExpression : 'Id= :Id',
FilterExpression : 'Id in (:Id)',
ExpressionAttributeValues: {
":Id": event.Id
}
};
mydocumentClient.scan(params, function (err, data){
if (err) {
callback(err, null);
}else{
callback(null, data);
}
})
}
I have a Lambda with the following code:
// Load the AWS SDK for Node.js.
var AWS = require("aws-sdk");
// Set the AWS Region.
AWS.config.update({ region: "us-east-2" });
exports.handler = function(event, context, callback) {
// Create DynamoDB service object.
var ddb = new AWS.DynamoDB({ apiVersion: "2012-08-10" });
var params = {
TableName: "Ranking",
ProjectionExpression: "#username, Score, Duration",
FilterExpression: "#username = :username",
ExpressionAttributeNames: {
"#username": "Username",
},
ExpressionAttributeValues: {
":username": {
S: 'Alberto'
},
}
};
let toReturn = [];
ddb.scan(params, function (err, data) {
if (err) {
toReturn = err
} else {
toReturn = data.Items;
}
});
let response = {
statusCode: 200,
body: JSON.stringify(toReturn)
};
callback(null, response)
};
However I always see [] as response...
My current DB has the following records:
So my question is... why I don't get back that item?
Since you are using callbacks, your code should be as below. Also Duration is reserved keyword, so it also needs to be modified as below:
// Load the AWS SDK for Node.js.
var AWS = require("aws-sdk");
// Set the AWS Region.
AWS.config.update({ region: "us-east-2" });
exports.handler = function(event, context, callback) {
// Create DynamoDB service object.
var ddb = new AWS.DynamoDB({ apiVersion: "2012-08-10" });
var params = {
TableName: "Ranking",
FilterExpression: "#username = :username",
ProjectionExpression: "#username, Score, #duration",
ExpressionAttributeNames: {
"#username": "Username",
"#duration": "Duration",
},
ExpressionAttributeValues: {
":username": {
S: 'Alberto'
},
}
};
ddb.scan(params, function (err, data) {
if (err) {
callback(null, err)
} else {
callback(null, data.Items)
}
});
};
I use aws-sdk in nodejs, And I should scan database with wildcard.
I tried this with aws developer guide:
var params = {
TableName: RecipeTable,
FilterExpression: "#recipe = :recipe",
ExpressionAttributeNames:{
"#recipe": "recipe",
},
ExpressionAttributeValues: {
":recipe": request.params.recipe,
}
};
I can't reach the answer.
Can someone help me?
Thanks.
I found the answer: Using contains keyword.
app.get('/recipe/:recipe', (request, response) => {
var params = {
TableName: RecipeTable,
FilterExpression: "contains(#recipe, :recipe)", // Here!
ExpressionAttributeNames:{
"#recipe": "ingredients"
},
ExpressionAttributeValues: {
":recipe": request.params.recipe
}
};
result = [];
docClient.scan(params, onScan);
function onScan(err, data) {
if (!err) {
data.Items.forEach((itemdata) => {
result.push(itemdata);
});
if(typeof data.LastEvaluatedKey != "undefined") {
params.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.scan(params, onScan);
} else {
response.send(JSON.stringify({"result" : result}));
}
}
}
})
I have referenced this document.
Thanks.
I have this Lambda code setup in JS for inserting a row in DynamoDB but the code seems to have some issue and doesn't trigger.Unable to figure out the issue as cloud watch also does captures anything. The API Gateway triggers the Lambda will inserts a row in DynamoDB. Tried debugging with console and logs also but unable to figure out the actual issue that is causing the code to break.
exports.handler = async (event) => {
console.log(event);
var eventBody = JSON.parse(event.body);
var eventType = eventBody.event_type;
var content = eventBody.content;
if (content.subscription) {
var subscription = content.subscription;
switch (eventType) {
case "subscription_created":
// Add new entry to License table
console.log('subscription created event');
var customer = content.customer;
var findUserScan = {
ExpressionAttributeValues: {
':email': { S: customer.email },
},
FilterExpression: 'email = :email',
TableName: _dynamodb_userprofile_table
};
try {
await dynamodb.scan(findUserScan, function (err, data) {
if (err) {
console.log("Error", err);
} else {
var userProfileAWS;
data.Items.forEach(function (userProfile, index, array) {
userProfileAWS = AWS.DynamoDB.Converter.unmarshall(userProfile);
});
var companyName;
if (userProfileAWS) {
companyName = userProfileAWS.organization;
}
try {
var licenseEntry = {
"subscriptionID": {
S: subscription.id
},
"companyName": {
S: companyName || ""
},
"customerEmail": {
S: customer.email
},
};
await dynamodb.putItem({
"TableName": _dynamodb_license_table,
"Item": licenseEntry
}).promise()
} catch (error) {
console.log(error)
throw new Error(`Error in dynamoDB: ${JSON.stringify(error)}`);
}
}
}).promise();
} catch (error) {
throw new Error(`Error in dynamoDB: ${JSON.stringify(error)}`);
}
break;
So I need a recursive function in node.js for replacing this function call:
docClient.scan(params, callback)
More info see http://docs.aws.amazon.com/amazondynamodb/latest/gettingstartedguide/GettingStarted.NodeJs.04.html
Here is the recursive code to execute the scan until LastEvaluatedKey is available.
var AWS = require("aws-sdk");
var creds = new AWS.Credentials('akid', 'secret', 'session');
AWS.config.update({
region: "us-west-2",
endpoint: "http://localhost:8000",
credentials : creds
});
var docClient = new AWS.DynamoDB.DocumentClient();
var params = {
TableName: "Movies"
};
console.log("Scanning Movies table.");
docClient.scan(params, onScan);
var count = 0;
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
} else {
// print all the movies
console.log("Scan succeeded.");
data.Items.forEach(function(movie) {
console.log("Item :", ++count,JSON.stringify(movie));
});
// continue scanning if we have more movies
if (typeof data.LastEvaluatedKey != "undefined") {
console.log("Scanning for more...");
params.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.scan(params, onScan);
}
}
}
Recursive DynamoDB Scan using AWS SDK V3.
import {
DynamoDBClient,
ScanCommand,
ScanCommandOutput,
ScanCommandInput,
} from '#aws-sdk/client-dynamodb';
import { unmarshall, marshall } from '#aws-sdk/util-dynamodb';
const client = new DynamoDBClient({ region: 'eu-west-1' });
export const scanTable = async<ResultType>(params: ScanCommandInput): Promise<ResultType[]> => {
const scanParams: ScanCommandInput = params;
const results = [];
let lastEvaluatedKey: string;
do {
console.log(`Running query: ${JSON.stringify(scanParams)}`);
const { Items, LastEvaluatedKey } = await client.send(new ScanCommand(scanParams));
lastEvaluatedKey = LastEvaluatedKey;
Items.forEach((item) => results.push(unmarshall(item)));
scanParams.ExclusiveStartKey = LastEvaluatedKey;
} while (typeof lastEvaluatedKey !== 'undefined');
return results as ResultType[];
};
const items = await scanTable({ TableName: 'dynamo-table' })