Combining multiple queries in one aws lambda function - node.js

In my DynamoDB table I've got items with userA (MyUser) following UserB(myFriends..)
Primary-------|---- Srotkey---|---FriendID---|
Myid..... Friend-01 22223333
in the same table I also have the user profiles..
Primary-------|---- Srotkey---|----Name----|
Myid..... Friend-01
22223333 Profile Rose
Now I want to have a a function to return my friend's profile.
I suppose my lambda "getFriendProfile" function should do two queries, get the id for the person I'm following in the first query and then use that result to fetch her profile in the second query.
I know how to get those result individually but I don't know how to combine them and put them both into one function.
First query (getPeopleIFollow)
module.exports.handler = async (event, context, callback) => {
const _userID = event.userID;
var params = {
TableName: "mytableName",
ProjectionExpression: "FriendID",
KeyConditionExpression: "#tp = :userId and begins_with(#sk, :skv)",
ExpressionAttributeNames: {
"#tp": "userId",
"#sk": "Sortkey",
},
ExpressionAttributeValues: {
":userId": { "S": _userID },
":skv": { "S": "Friend" },
}
}
var Friend_ID;
try {
let data = await dynamodb.query(params).promise();
data.Items.map(
(dataField) => {
Friend_ID = dataField.FriendID.S,
}
);
callback(null, Friend_ID );
}
catch (error) {
console.log(error);
callback(error);
}
};
My other function looks just looks very much like the first one..
getProfileNames..
module.exports.handler = async (event, context, callback) => {
const _userID = event.myfriendID;
var params = {
TableName: "mytableName",
ProjectionExpression: "Name",
KeyConditionExpression: "#tp = :userId and begins_with(#sk, :skv)",
ExpressionAttributeNames: {
"#tp": "userId",
"#sk": "Sortkey",
},
ExpressionAttributeValues: {
":userId": { "S": _userID },
":skv": { "S": "Profile" },
}
}
try {
let data = await dynamodb.query(params).promise();
const items = data.Items.map(
(dataField) => {
return {
friend_name: dataField.Name.S,
}
}
);
callback(null, items );
}
catch (error) {
console.log(error);
callback(error);
}
};

Keeping functionality separate to each Lambda is actually a nice way to organise code. So you could keep your code as it is and invoke the getProfileNames lambda for each of the getPeopleIFollow results.
I think it's also a good idea to abstract as much as possible out of your handler so that it becomes easy to follow at a high level. So the following examples assumes your code is in other functions that return promises.
(untested code)
module.exports.handler = async (event, context, callback) => {
const _userID = event.userID;
try {
const userFriends = await getUserFriends(userId);
const friendProfiles = await getFriendProfiles(userFriends);
callback(null, friendProfiles );
} catch (error) {
console.log(error);
callback(error);
}
}
The lambda invocation happens in getFriendProfiles which uses Promise.all to invoke lambdas for each separate friendProfile request.
(untested code)
function getFriendProfiles(userFriends){
return Promise.all(userFriends.map(userFriend => {
const params = {
ClientContext: "MyApp",
FunctionName: "MyFunction",
InvocationType: "RequestResponse", // important, you choose this so you get a response otherwise (eg if you choose the 'Event' InvocationType) I think the promise will resolve once the invocation is successful
LogType: "Tail",
Payload: JSON.stringify({ myfriendID: userFriend }), // I think this is right, I usually use pathParameters or body for my payloads
Qualifier: "1"
};
lambda.invoke(params).promise();
});
}
Having said all that - what this number of db queries shows is that perhaps this DynamoDb schema is not the most efficient. This might be a case where indexes could be helpful, or even using an RDS rather than dynamodb.

Related

Aws Lambda to read the user input dynamically and return the data from Dynamo table

Lambda provides the expected result only when I pass the value manually (Id = '011010').
In the step function the value "ID" value will be random based on the logic from previous step, since the Id value is not static how to handle this scenario "ExpressionAttributeValues"
I tried all the below syntax but no luck..
ExpressionAttributeValues: { ':value': $.External.Id}
ExpressionAttributeValues: { ':value': External.Id.$}
ExpressionAttributeValues: { ':value': $.Id}
ExpressionAttributeValues: { ':value': Id.$}
ExpressionAttributeValues: { ':value': event.Id}
Lambda-Code
'use strict'
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient();
var params = {
TableName: 'temptable',
IndexName: 'Id-CurrentStatus-index',
KeyConditionExpression: '#Id= :value',
ExpressionAttributeNames: { '#Id': 'Id'},
ExpressionAttributeValues: { ':value': 'M1' }
};
async function queryItems(){
try {
const data = await docClient.query(params).promise()
return data
} catch (err) {
return err
}
}
exports.handler = async (event, context) => {
try {
const data = await queryItems()
return { body: JSON.stringify(data) }
} catch (err) {
return { error: err }
}
}
I can read it from the
console.log("Memberid :" + JSON.stringify(event.Id, null, 2))
but how to pass the same value in the
ExpressionAttributeValues: { ':value': 'M1' }
I tried the below syntax. nothing works
ExpressionAttributeValues: { ':value': JSON.stringify(event.Id, null, 2) }
ExpressionAttributeValues: { ':value': event.Id}
ExpressionAttributeValues: { ':value': event.Id}
ExpressionAttributeValues: { ':value': Id}
I got the required code after checking multiple thread.
AWS Lambda function to scan/query DynamoDB table using array values as FilterExpression
'use strict'
var AWS = require('aws-sdk');
var mydocumentClient = new AWS.DynamoDB.DocumentClient();
exports.handler = function (event, context, callback) {
var params = {
TableName: 'temptable',
KeyConditionExpression : 'Id= :Id',
FilterExpression : 'Id in (:Id)',
ExpressionAttributeValues: {
":Id": event.Id
}
};
mydocumentClient.scan(params, function (err, data){
if (err) {
callback(err, null);
}else{
callback(null, data);
}
})
}

Data Item not getting inserted in DynamoDB Table from AWS Lambda Trigger

I have this Lambda code setup in JS for inserting a row in DynamoDB but the code seems to have some issue and doesn't trigger.Unable to figure out the issue as cloud watch also does captures anything. The API Gateway triggers the Lambda will inserts a row in DynamoDB. Tried debugging with console and logs also but unable to figure out the actual issue that is causing the code to break.
exports.handler = async (event) => {
console.log(event);
var eventBody = JSON.parse(event.body);
var eventType = eventBody.event_type;
var content = eventBody.content;
if (content.subscription) {
var subscription = content.subscription;
switch (eventType) {
case "subscription_created":
// Add new entry to License table
console.log('subscription created event');
var customer = content.customer;
var findUserScan = {
ExpressionAttributeValues: {
':email': { S: customer.email },
},
FilterExpression: 'email = :email',
TableName: _dynamodb_userprofile_table
};
try {
await dynamodb.scan(findUserScan, function (err, data) {
if (err) {
console.log("Error", err);
} else {
var userProfileAWS;
data.Items.forEach(function (userProfile, index, array) {
userProfileAWS = AWS.DynamoDB.Converter.unmarshall(userProfile);
});
var companyName;
if (userProfileAWS) {
companyName = userProfileAWS.organization;
}
try {
var licenseEntry = {
"subscriptionID": {
S: subscription.id
},
"companyName": {
S: companyName || ""
},
"customerEmail": {
S: customer.email
},
};
await dynamodb.putItem({
"TableName": _dynamodb_license_table,
"Item": licenseEntry
}).promise()
} catch (error) {
console.log(error)
throw new Error(`Error in dynamoDB: ${JSON.stringify(error)}`);
}
}
}).promise();
} catch (error) {
throw new Error(`Error in dynamoDB: ${JSON.stringify(error)}`);
}
break;

Cannot return array if item not present in dynamodb

I have a function that will take an array of jobs as a parameter in it. This function will check the existence of each job in the database through its id.
If a job is not to present in the database, that particular job needs to be pushed into an array called latestJobs. I'm calling this function in my main.js file. But the code breaks and stops.
Below is my main.js code:
module.exports.app = async () => {
try {
...
const jobs = await getJobsForCountries(body);
const latestJobs = await filterPreDraftedJobs(jobs);
console.log('latestJobs', latestJobs);
} catch (e) {
console.error('Error:- ', e); // Comes to here
}
};
My checker function looks like:
module.exports = async (jobs) => {
let latestJobs = [];
for (const job of jobs) {
const params = {
TableName: process.env.DYNAMODB_TABLE,
Key: {
id: job.Id
}
};
await dynamoDb.get(params, (err, data) => {
if (err) {
latestJobs.push(job);
console.log('Job not found in DB');
}
}).promise();
}
return latestJobs;
};
How can I fix this issue? I want the latestJobs which will not present in the database. Is there a function for dynamodb which can do this for me?
You are mixing callback, promise and await style. I would do it like this
module.exports = async (jobs) => {
let latestJobs = [];
for (const job of jobs) {
const params = {
TableName: process.env.DYNAMODB_TABLE,
Key: {
id: job.Id
}
};
try {
const result = await dynamoDb.get(params).promise();
if (result) {
return;
}
} catch(err) {
latestJobs.push(job);
}
}
return latestJobs;
};
Also, make sure that table is created and the region and name you are passing is correct.
I am not much familiar with dynamoDB but looking at the above conversation code must be something like this. I have tried to improve performance and making sure the code is modular and readable.
async function addUpdateJobs(jobs)
{
let paramsArray = [];
for (const job of jobs)
{
const jobParams = {
params:{
TableName: process.env.DYNAMODB_TABLE,
Key: {
id: job.Id
}
},
job:job
};
paramsArray.push(jobParams );
}
return await this.getJobs(paramsArray);
}
function getJobs(paramsArray)
{
let latestJobs = [];
paramsArray.each(async (jobParam)=>
{
try
{
const result = await dynamoDb.get(jobParam.params).promise();
if (result)
{
return;
}
} catch (err)
{
latestJobs.push(jobParam.job);
}
});
return latestJobs;
}
PS: Also I was gonig through error handling in amazondynamodb.

Alexa skill async await fetching data from DynamoDB

The following code is my launch handler in my Alexa skill and I have a variable named x inside of my handler. I am trying to set x to data that I'm getting from dynamoDB and to use it outside of the get function (I got the function from https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GettingStarted.NodeJs.03.html#GettingStarted.NodeJs.03.02) so that Alexa can speak the value (a string) of x (as you see in the return). The statement in my "get" function is not changing the value of x outside of the get function itself. I know that the x inside of the get function is actually being changed because I am logging it to console. So I posted a similar post on this, and I initially thought it was a scoping issue, but turns out it's because the get function is asynchronous. Hence, I added the async and await keywords as shown below. I'm new to NodeJS so that's where I thought I should put them, according to what I've researched. This is still not working however.
const LaunchHandler = {
canHandle(handlerInput) {
return handlerInput.requestEnvelope.request.type === `LaunchRequest`;
},
async handle(handlerInput) {
var x;
//DYNAMO GET FUNCTION
await DBClient.get(params, function(err, data) {
if (err) {
console.error("Unable to read item. Error JSON:", JSON.stringify(err, null, 2));
} else {
x = data.Item.Answer;
} });
return handlerInput.responseBuilder
.speak(x)
.withShouldEndSession(false)
.getResponse();
},
};
As a side note, here's the JSON that I'm (successfully) returning from the database:
{
"Item": {
"Answer": "Sunny weather",
"Question": "What is the weather like today"
}
}
Are you looking for something like this? Within the handle function i call another function getSpeechOutput to create some feedback text. Thus function calls the dynamodb function getGA to get user data
const getSpeechOutput = async function (version) {
const gadata = await ga.getGA(gaQueryUsers, 'ga:users')
let speechOutput;
...
return ...
}
const UsersIntentHandler = {
canHandle(handlerInput) {
return handlerInput.requestEnvelope.request.type === 'IntentRequest'
&& handlerInput.requestEnvelope.request.intent.name === 'UsersIntent';
},
async handle(handlerInput) {
try {
let speechOutput
...
speechOutput = await getSpeechOutput("long");
...
return handlerInput.responseBuilder
.speak(speechOutput)
.reprompt("Noch eine Frage?")
.withSimpleCard(defaulttext.SKILL_NAME, speechOutput)
.getResponse();
} catch (error) {
console.error(error);
}
},
};
Thats the db function:
const getUser = async function (userId) {
const dynamodbParams = {
TableName: process.env.DYNAMODB_TABLE_BLICKANALYTICS,
Key: {
id: userId
}
}
return await dynamoDb.get(dynamodbParams).promise()
}

Delete all items in Dynamodb using Lambda?

Using Lambda (node.js) - how to delete all the items in the Dynamodb table?
There are 500K rows in the table
I have tried using scan method and then loop through each item and then using delete method. It only allow up to 3000 rows only.
Code
exports.handler = function(context, callback) {
getRecords().then((data) => {
data.Items.forEach(function(item) {
deleteItem(item.Id).then((data1) => {
});
});
});
};
var deleteItem = function(id) {
var params = {
TableName: "TableName",
Key: {
"Id": id
},
};
return new Promise(function(resolve, reject) {
client.delete(params, function(err, data) {
if (err) {
reject(err);
} else {
resolve();
}
});
});
}
function getRecords() {
var params = {
TableName: 'TableName',
IndexName: 'Type-index',
KeyConditionExpression: 'Type = :ty',
ExpressionAttributeValues: {
':ty': "1"
},
ProjectionExpression: "Id",
};
return new Promise(function(resolve, reject) {
client.query(params, function(err, data) {
if (err) {
reject(err);
} else {
resolve(data);
}
});
});
}
There is already one right answer, but here is another code snippet to delete all records from Dynamo DB.
const AWS = require("aws-sdk");
AWS.config.update({
region: "us-east-1",
});
const docClient = new AWS.DynamoDB.DocumentClient();
const getAllRecords = async (table) => {
let params = {
TableName: table,
};
let items = [];
let data = await docClient.scan(params).promise();
items = [...items, ...data.Items];
while (typeof data.LastEvaluatedKey != "undefined") {
params.ExclusiveStartKey = data.LastEvaluatedKey;
data = await docClient.scan(params).promise();
items = [...items, ...data.Items];
}
return items;
};
const deleteItem = (table, id) => {
var params = {
TableName: table,
Key: {
id: id,
},
};
return new Promise(function (resolve, reject) {
docClient.delete(params, function (err, data) {
if (err) {
console.log("Error Deleting ", id,err);
reject(err);
} else {
console.log("Success Deleting ", id,err);
resolve();
}
});
});
};
exports.handler = async function (event, context, callback) {
try {
const tableName = "<table>";
// scan and get all items
const allRecords = await getAllRecords(tableName);
// delete one by one
for (const item of allRecords) {
await deleteItem(tableName, item.id);
}
callback(null, {
msg: "All records are deleted.",
});
} catch (e) {
callback(null, JSON.stringify(e, null, 2));
}
};
A Scan operation consumes Read capacity. Each Read returns up to 4 kb of data. When this limit is reached, the Scan returns only what it has found until there. If you need more, you need to issue another Scan request.
This, you'll need two loops: 1) loop to delete all records returned at each Scan; 2) loop to keep scanning multiple times, until you reach the end of the table
Make sure you use consistent Reads or wait 1 or 2 second(s) before issuing another Scan, otherwise you may get repeated items in different Scans.
exports.handler = function(context, callback) {
clearRecords();
};
clearRecords = function() {
getRecords().then((data) => {
data.Items.forEach(function(item) {
deleteItem(item.Id).then((data1) => {});
});
clearRecords(); // Will call the same function over and over
});
}
Observe that Lambda has a timeout limit of 15 minutes. Since you have 500K items in your table, it's likely that your Lambda will timeout and you'll need to trigger it more than once. You could also make your Lambda call itself after 14:50, for example, just take a look at the AWS SDK documentation for triggering Lambda functions. For this matter, you might also want to check the getRemainingTimeInMillis() method from the context object.

Resources