How to update() DynamoDB - node.js

I'm working with DynamoDB for the first time and so far, not too bad. Having some understanding of Node is making this easier, but having none of noSQL, well that hurts!. My application relies on quickly searching a GROUPS list and returning members from that group.
When I add the data using the console, I am able to create a JSON document like this:
"GROUP_ID":"GroupA", "MEMBER": ["MemberA", "MemberB", "MemberC"], "STATUS": ["OWNER", "MEMBER", "INVITED"], "ADD_DATE":[1234567, 2345671, 3456712]
I am using this to create the GROUP_ID as the primary key:
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({region:'us-east-1'});
exports.handler = (event, context, callback) => {
var tableName = "USER_GROUPS";
var checkParams = {
TableName: tableName,
Key:{"GROUP_NAME": event.group_id}
};
var createParams = {
TableName: tableName,
Key:{"GROUP_NAME": event.group_id},
Item:{"GROUP_NAME": event.group_id, "MEMBER_NAME":[event.device_id], "MEMBER_SATUS":["OWNER"], "MEMBER_DATE" : [event.date]}
};
var checkGroupExists = new Promise((resolve, reject) => {
docClient.get(checkParams, (err, data) => {
if(err){
reject(err);
}
if(data.Item){ // exists...
console.log("found Group ID");
reject();
}
else{
console.log("Group ID not found...");
resolve();
}
});
});
checkGroupExists.then((err, data) => {
console.log("adding Group: ");
console.log(data);
docClient.put(createParams, (err, data) =>{
if(err){
console.log(err);
callback(err, null);
}else{
console.log("added device: " + createParams.Key);
callback(null, {"created":createParams.Key.GROUP_NAME});
}
});
}).catch((err) => {
if(err){
callback(err, null);
}
else{
callback(err, {"used":createParams.Key.GROUP_NAME});
}
});
};
but when I use update() I am adding to the array and end up with a type name for the arrays
["S":"MemberA", "S":"MemberB", "S":"MemberC"]
kind of thing....
This is where I have left off, quite frustrated here!
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({region:'us-east-1'});
exports.handler = (event, context, callback) => {
var tableName = "USER_GROUPS";
var checkParams = {
TableName: tableName,
Key:{"GROUP_NAME": event.group_id},
Item:{"MEMBER_NAME": event.member_name}
};
var updateParams = {
TableName: tableName,
Key:{"GROUP_NAME": event.group_id},
//Item:{"MEMBER_NAME":[event.device_id], "MEMBER_STATUS":["OWNER"], "MEMBER_DATE" : [event.date]},
AttributeUpdates: {
"MEMBER_NAME": {
Action: 'ADD',
Value: event.device_id
},
"MEMBER_STATUS":{
Action : 'ADD',
Value: "ENVITED"
},
"MEMBER_DATE":{
Action: 'ADD',
Value: event.date
}
},
};
var checkGroupExists = new Promise((resolve, reject) => {
docClient.get(checkParams, (err, data) => {
console.log(data);
if(err){
reject(err);
}
if(data.Item.GROUP_NAME){ // exists...
console.log("found the group...");
resolve();
}
else{
reject();
}
});
});
checkGroupExists.then((err, data) => {
docClient.update(updateParams, (err, data) =>{
if(err){
console.log(err);
callback(err, null);
}else{
callback(null, {"updated" : updateParams.Key.GROUP_NAME});
}
});
}).catch((err) => {
if(err){
callback(err, null);
}
else{
callback(err, {"noGroup":createParams.Key.GROUP_NAME});
}
});
};
It seems I am lacking the understanding of how to process updates to include NEW item in the member array. Any assistance would be appreciated!

Use list_append() together in an UpdateExpression to append to a list column along with a ConditionExpression to prevent updates to a non-existent group:
const AWS = require('aws-sdk');
const DB = new AWS.DynamoDB.DocumentClient({region:'us-east-1'});
const UpdateExpression = 'SET ' + [
'#members = list_append(#members, :member)',
'#statuses = list_append(#statuses, :status)',
'#dates = list_append(#dates, :date)',
].join(', ')
const ConditionExpression = [
'attribute_type(#members, :L)',
'attribute_type(#statuses, :L)',
'attribute_type(#dates, :L)'
].join(' AND ')
const ExpressionAttributeNames = {
'#members': 'MEMBER_NAME',
'#statuses': 'MEMBER_STATUS',
'#dates': 'MEMBER_DATE'
}
exports.handler = (event, context, callback) => {
DB.update({
TableName: 'USER_GROUPS',
Key: { GROUP_NAME: e.group_id },
ReturnValues: 'ALL_NEW',
UpdateExpression: UpdateExpression,
ConditionExpression: ConditionExpression,
ExpressionAttributeNames: ExpressionAttributeNames,
ExpressionAttributeValues: {
':member': [e.member_name],
':status': ['ENVITED'],
':date': [e.date],
':L': 'L'
}
}, callback)
}

Related

Data Item not getting inserted in DynamoDB Table from AWS Lambda Trigger

I have this Lambda code setup in JS for inserting a row in DynamoDB but the code seems to have some issue and doesn't trigger.Unable to figure out the issue as cloud watch also does captures anything. The API Gateway triggers the Lambda will inserts a row in DynamoDB. Tried debugging with console and logs also but unable to figure out the actual issue that is causing the code to break.
exports.handler = async (event) => {
console.log(event);
var eventBody = JSON.parse(event.body);
var eventType = eventBody.event_type;
var content = eventBody.content;
if (content.subscription) {
var subscription = content.subscription;
switch (eventType) {
case "subscription_created":
// Add new entry to License table
console.log('subscription created event');
var customer = content.customer;
var findUserScan = {
ExpressionAttributeValues: {
':email': { S: customer.email },
},
FilterExpression: 'email = :email',
TableName: _dynamodb_userprofile_table
};
try {
await dynamodb.scan(findUserScan, function (err, data) {
if (err) {
console.log("Error", err);
} else {
var userProfileAWS;
data.Items.forEach(function (userProfile, index, array) {
userProfileAWS = AWS.DynamoDB.Converter.unmarshall(userProfile);
});
var companyName;
if (userProfileAWS) {
companyName = userProfileAWS.organization;
}
try {
var licenseEntry = {
"subscriptionID": {
S: subscription.id
},
"companyName": {
S: companyName || ""
},
"customerEmail": {
S: customer.email
},
};
await dynamodb.putItem({
"TableName": _dynamodb_license_table,
"Item": licenseEntry
}).promise()
} catch (error) {
console.log(error)
throw new Error(`Error in dynamoDB: ${JSON.stringify(error)}`);
}
}
}).promise();
} catch (error) {
throw new Error(`Error in dynamoDB: ${JSON.stringify(error)}`);
}
break;

node.js module answer undefined [duplicate]

This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 4 years ago.
Can someone give me a hand? I tried to figure it out but i ran out of ideas.
-------------dynamo.js....
module.exports.readUser = function (user_id) {
AWS.config = new AWS.Config();
AWS.config.update({region: "eu-west-1"});
var docClient = new AWS.DynamoDB.DocumentClient();
var table = "user";
var user_id = user_id;
var params = {
TableName: table,
Key:{
"user_id": user_id
}
};
docClient.get(params, function(err, data) {
if (err) {
return err;
} else {
//console.log(data); <-- data is filled
return data;
}
});
var dynamo = require("./dynamo.js");
console.log(dynamo.readUser(4711998));
The data you log out inside the function is only the returned value for the anonymous function given to docClient.get as the second argument. The exported readUser function has no return statement and therefore it is undefined.
Because of the async nature of javascript you have to access the result via a callback like so:
module.exports.readUser = function (user_id, cb) {
...
docClient.get(params, function(err, data) {
if (err) {
return err;
} else {
return cb(data);
}
});
}
var dynamo = require("./dynamo.js");
dynamo.readUser(4711998, function (user) {
console.log(user);
});
docClient.get is an asynchronous function. You need promisify this function or use callback.
Use callback:
module.exports.readUser = function (user_id, cb) {
AWS.config = new AWS.Config();
AWS.config.update({ region: "eu-west-1" });
var docClient = new AWS.DynamoDB.DocumentClient();
var table = "user";
var user_id = user_id;
var params = {
TableName: table,
Key: {
"user_id": user_id
}
};
docClient.get(params, cb);
}
var dynamo = require("./dynamo.js");
dynamo.readUser(4711998, function (err, user) {
if(err)
console.error(err);
else
console.log(user);
});
Use promise:
module.exports.readUser = function (user_id) {
AWS.config = new AWS.Config();
AWS.config.update({ region: "eu-west-1" });
var docClient = new AWS.DynamoDB.DocumentClient();
var table = "user";
var user_id = user_id;
var params = {
TableName: table,
Key: {
"user_id": user_id
}
};
return new Promise(function(resolve, reject) {
docClient.get(params, function(err, data) {
if (err) {
reject(err);
} else {
resolve(data);
}
});
});
}
var dynamo = require("./dynamo.js");
dynamo.readUser(4711998).then(function (user) {
console.log(user);
}).catch(function(err) {
console.error(err);
});
Read more:
How do I promisify the AWS JavaScript SDK?
https://aws.amazon.com/ru/blogs/developer/support-for-promises-in-the-sdk/

Delete all items in Dynamodb using Lambda?

Using Lambda (node.js) - how to delete all the items in the Dynamodb table?
There are 500K rows in the table
I have tried using scan method and then loop through each item and then using delete method. It only allow up to 3000 rows only.
Code
exports.handler = function(context, callback) {
getRecords().then((data) => {
data.Items.forEach(function(item) {
deleteItem(item.Id).then((data1) => {
});
});
});
};
var deleteItem = function(id) {
var params = {
TableName: "TableName",
Key: {
"Id": id
},
};
return new Promise(function(resolve, reject) {
client.delete(params, function(err, data) {
if (err) {
reject(err);
} else {
resolve();
}
});
});
}
function getRecords() {
var params = {
TableName: 'TableName',
IndexName: 'Type-index',
KeyConditionExpression: 'Type = :ty',
ExpressionAttributeValues: {
':ty': "1"
},
ProjectionExpression: "Id",
};
return new Promise(function(resolve, reject) {
client.query(params, function(err, data) {
if (err) {
reject(err);
} else {
resolve(data);
}
});
});
}
There is already one right answer, but here is another code snippet to delete all records from Dynamo DB.
const AWS = require("aws-sdk");
AWS.config.update({
region: "us-east-1",
});
const docClient = new AWS.DynamoDB.DocumentClient();
const getAllRecords = async (table) => {
let params = {
TableName: table,
};
let items = [];
let data = await docClient.scan(params).promise();
items = [...items, ...data.Items];
while (typeof data.LastEvaluatedKey != "undefined") {
params.ExclusiveStartKey = data.LastEvaluatedKey;
data = await docClient.scan(params).promise();
items = [...items, ...data.Items];
}
return items;
};
const deleteItem = (table, id) => {
var params = {
TableName: table,
Key: {
id: id,
},
};
return new Promise(function (resolve, reject) {
docClient.delete(params, function (err, data) {
if (err) {
console.log("Error Deleting ", id,err);
reject(err);
} else {
console.log("Success Deleting ", id,err);
resolve();
}
});
});
};
exports.handler = async function (event, context, callback) {
try {
const tableName = "<table>";
// scan and get all items
const allRecords = await getAllRecords(tableName);
// delete one by one
for (const item of allRecords) {
await deleteItem(tableName, item.id);
}
callback(null, {
msg: "All records are deleted.",
});
} catch (e) {
callback(null, JSON.stringify(e, null, 2));
}
};
A Scan operation consumes Read capacity. Each Read returns up to 4 kb of data. When this limit is reached, the Scan returns only what it has found until there. If you need more, you need to issue another Scan request.
This, you'll need two loops: 1) loop to delete all records returned at each Scan; 2) loop to keep scanning multiple times, until you reach the end of the table
Make sure you use consistent Reads or wait 1 or 2 second(s) before issuing another Scan, otherwise you may get repeated items in different Scans.
exports.handler = function(context, callback) {
clearRecords();
};
clearRecords = function() {
getRecords().then((data) => {
data.Items.forEach(function(item) {
deleteItem(item.Id).then((data1) => {});
});
clearRecords(); // Will call the same function over and over
});
}
Observe that Lambda has a timeout limit of 15 minutes. Since you have 500K items in your table, it's likely that your Lambda will timeout and you'll need to trigger it more than once. You could also make your Lambda call itself after 14:50, for example, just take a look at the AWS SDK documentation for triggering Lambda functions. For this matter, you might also want to check the getRemainingTimeInMillis() method from the context object.

Unable to retrive data and push inside loop in node js

I am trying to retrieve attendance list along with user details.
I am using caminte.js(http://www.camintejs.com/) Cross-db ORM for database interaction.
Here is my code sample of model function "attendanceList".
exports.attendanceList = function (req, callback) {
var query = req.query;
var searchfilters = {};
if(!req.user){
callback({ code:400, status:'error', message: 'Invalid Request', data:{}});
}else{
searchfilters["vendor_id"] = parseInt(req.user._id);
}
if(query.location && parseString(query.location) != '') {
searchfilters["location"] = parseString(query.location);
}
if (query.device_details && parseString(query.device_details) != '') {
searchfilters["device_details"] = parseString(query.device_details);
}
if(query.created_on) {
searchfilters["created_on"] = query.created_on;
}
if(query.status) {
searchfilters["status"] = { regex: new RegExp(query.status.toLowerCase(), "i") };
}
var SkipRecord = 0;
var PageSize = 10;
var LimitRecord = PageSize;
var PageIndex = 1;
if(query.pagesize) {
PageSize = parseInt(query.pagesize);
}
if(query.pageindex) {
PageIndex = parseInt(query.pageindex);
}
if (PageIndex > 1) {
SkipRecord = (PageIndex - 1) * PageSize;
}
LimitRecord = PageSize;
var SortRecord = "created_on";
if(query.sortby && query.sorttype) {
var sortingBy = query.sortby;
var sortingType = 'ASC';
if(typeof query.sorttype !== 'undefined') {
sortingType = query.sorttype;
}
SortRecord = sortingBy + ' ' + sortingType;
}
Attendance.find({ where: searchfilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, async function (err, result) {
if(err){
callback({ code:400, status:'error', message:'Unable to connect server', errors:err });
} else {
await result.map(function(row, i){
User.findById(parseInt(row.user_id), function(err, data){
if(err){
console.log(err);
} else {
result[i]['userDetails'] = data;
}
});
});
await Attendance.count({ where: searchfilters }, function (err, count) {
callback({ code:200, status:'success', message:'OK', total:count, data:result });
});
}
});
};
I am getting only attendance list without user details. How do I force to push user details into attendance list? Any Help!!
Thank You
This behavior is asynchronous. When you're making request to DB, your code keeps running, while task to get data comes to task queue.
To keep things simple, you need to use promises while handling asynchronous jobs.
Rewrite your code from this:
Attendance.find({ where: searchfilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, async function (err, result) {
if(err){
callback({ code:400, status:'error', message:'Unable to connect server', errors:err });
} else {
await result.map(function(row, i){
User.findById(parseInt(row.user_id), function(err, data){
if(err){
console.log(err);
} else {
result[i]['userDetails'] = data;
}
});
});
await Attendance.count({ where: searchfilters }, function (err, count) {
callback({ code:200, status:'success', message:'OK', total:count, data:result });
});
}
});
To this:
const findAttendanceFirst = (searchFilters, SortRecord, LimitRecord, SkipRecord) => {
return new Promise((resolve, reject) => {
Attendance.find({ where: searchFilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, (err, result) => {
if(err) return reject(err);
resolve(result);
});
});
}
const findUserByIdForUserDetails = (userId) => {
return new Promise((resolve, reject) => {
User.findById(parseInt(userId), function(err, data){
if(err) return reject(err);
resolve(data);
})
});
}
const getAttendanceCount = (searchFilters) => {
return new Promise((resolve, reject) => {
Attendance.count({ where: searchFilters }, (err, count) => {
if(err) return reject(err);
resolve(count);
});
})
}
So, now we can use this separate functions to make async behavior looks like sync.
try {
const data = await findAttendanceFirst(searchFilters, SortRecord, LimitRecord, SkipRecord);
for(let userData of data){
try {
userData.userDetails = await findUserByIdForUserDetails(userData.user_id);
} catch(e) {
// Some error happened, so no user details.
// you can set here null or nothing to userDetails.
}
}
let count;
try {
count = await getAttendanceCount(searchFilters);
} catch(e){
// Same as before.
}
const callBackData = { code:200, status:'success', message:'OK', total:count, data:result };
// And here you can do whatever you want with callback data. Send to client etc.
} catch(e) {
}
NB: I've not tested this code, it will be easier for yu to play with your actual data and use Promises and async/await
Just remember that each request to db is asynchronous, and you need to make your code wait for this data.

node js mongo db dependencies (doc not being found)

I have the following code:
var method = PushLoop.prototype;
var agent = require('./_header')
var request = require('request');
var User = require('../models/user_model.js');
var Message = require('../models/message_model.js');
var async = require('async')
function PushLoop() {};
method.startPushLoop = function() {
getUserList()
function getUserList() {
User.find({}, function(err, users) {
if (err) throw err;
if (users.length > 0) {
getUserMessages(users)
} else {
setTimeout(getUserList, 3000)
}
});
}
function getUserMessages(users) {
// console.log("getUserMessages")
async.eachSeries(users, function (user, callback) {
var params = {
email: user.email,
pwd: user.password,
token: user.device_token
}
messageRequest(params)
callback();
}, function (err) {
if (err) {
console.log(err)
setTimeout(getUserList, 3000)
}
});
}
function messageRequest(params) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ params.email +"&api_password="+ params.pwd +"&method=getSMS&type=1&limit=5"
request(url, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
if (responseObject["status"] == "success") {
async.eachSeries(messages, function(message, callback){
console.log(params.token)
saveMessage(message, params.token)
callback();
}, function(err) {
if (err) {
console.log(err)
}
// setTimeout(getUserList, 3000)
})
} else {
// setTimeout(getUserList, 3000)
}
} else {
console.log(err)
// setTimeout(getUserList, 3000)
}
});
setTimeout(getUserList, 3000)
}
function saveMessage(message, token) {
// { $and: [ { price: { $ne: 1.99 } }, { price: { $exists: true } }
// Message.find({ $and: [{ message_id: message.id}, {device_token: token}]}, function (err, doc){
Message.findOne({message_id: message.id}, function (err, doc){
if (!doc) {
console.log('emtpy today')
var m = new Message({
message_id: message.id,
did: message.did,
contact: message.contact,
message: message.message,
date: message.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: token
});
m.save(function(e) {
if (e) {
console.log(e)
} else {
agent.createMessage()
.device(token)
.alert(message.message)
.set('contact', message.contact)
.set('did', message.did)
.set('id', message.id)
.set('date', message.date)
.set('message', message.message)
.send();
}
});
}
}) //.limit(1);
}
};
module.exports = PushLoop;
Which actually works perfectly fine in my development environment - However in production (i'm using Openshift) the mongo documents get saved in an endless loop so it looks like the (if (!doc)) condition always return true therefore the document gets created each time. Not sure if this could be a mongoose issue - I also tried the "find" method instead of "findOne". My dev env has node 0.12.7 and Openshift has 0.10.x - this could be the issue, and i'm still investigating - but if anybody can spot an error I cannot see in my logic/code please let me know
thanks!
I solved this issue by using a "series" like pattern and using the shift method on the users array. The mongoose upsert findOneOrCreate is good however if there is a found document, the document is returned, if one isn't found and therefore created, it's also returned. Therefore I could not distinguish between the newly insert doc vs. a found doc, so used the same findOne function which returns null if no doc is found I just create it and send the push notification. Still abit ugly, and I know I could have used promises or the async lib, might refactor in the future. This works for now
function PushLoop() {};
var results = [];
method.go = function() {
var userArr = [];
startLoop()
function startLoop() {
User.find({},function(err, users) {
if (err) throw err;
users.forEach(function(u) {
userArr.push(u)
})
function async(arg, callback) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ arg.email +"&api_password="+ arg.password +"&method=getSMS&type=1&limit=5"
request.get(url, {timeout: 30000}, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
var status = responseObject.status
if (status === "success") {
messages.forEach(function(m) {
var message = new Message({
message_id: m.id,
did: m.did,
contact: m.contact,
message: m.message,
date: m.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: arg.device_token
});
var query = { $and : [{message_id: m.id}, {device_token: arg.device_token}] }
var query1 = { message_id: m.id }
Message.findOne(query).lean().exec(function (err, doc){
if (!doc || doc == null) {
message.save(function(e) {
console.log("message saved")
if (e) {
console.log("there is an error")
console.log(e)
} else {
console.log(message.device_token)
var messageStringCleaned = message.message.toString().replace(/\\/g,"");
var payload = {
"contact" : message.contact,
"did" : message.did,
"id" : message.message_id,
"date" : message.date,
"message" : messageStringCleaned
}
var note = new apns.Notification();
var myDevice = new apns.Device(message.device_token);
note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now.
note.badge = 3;
note.alert = messageStringCleaned;
note.payload = payload;
apnsConnection.pushNotification(note, myDevice);
}
})
}
});
});
}
else {
console.log(err)
}
}
});
setTimeout(function() {
callback(arg + "testing 12");
}, 1000);
}
// Final task (same in all the examples)
function series(item) {
if(item) {
async( item, function(result) {
results.push(result);
return series(userArr.shift());
});
} else {
return final();
}
}
function final() {
console.log('Done');
startLoop();
}
series(userArr.shift())
});
}
}
module.exports = PushLoop;

Resources