Aws lambda async and sync issue - node.js

I have a requirement where i need to query on a table then loop through the result and query another table inside a loop .. But i tried using synchronize node js module . But sometime api ends with timeout error with this method. Now i am trying my code in below way but no luck.
var notification = {
TableName: "Tablename",
KeyConditionExpression:"#userId =:userid and #datetime <=:datetime",
ScanIndexForward: false,
ExpressionAttributeNames: {
"#userId": "userId",
"#datetime":"datetime"
},
ExpressionAttributeValues: {
":userid" :userId,
":datetime" :datetime
}
};
docClient.query(notification, function(err, result) {
if (err) {
context.succeed({success: false, message : JSON.stringify(err, null, 2),method:"notification",type:type});
}else{
result = result.Items;
if (result.length>0) {
for(var i=0;i<result.length;i++){
result[k]['CafeDetail']=getDetail(result[k].CafeID,function (data) {
});
if (k ==(result.length -1)) {
context.succeed({success: true,data:result, message : "Notification list",method:"notification",type:type});
}
}
}
}
});
function getDetail(CafeID,callback) {
var param = {
TableName: "Tablename",
ProjectionExpression:"CafeName,Cafe_MainImage",
KeyConditionExpression:"CafeID =:cafeID",
ExpressionAttributeValues: {
":cafeID" :CafeID
}
};
docClient.query(param, function(err, CafeDetail) {
if (err) {
console.log (err)
callback(err);
} else {
callback(CafeDetail.Items);
console.log(CafeDetail.Items);
}
});
}
On output this variable (result[k]['CafeDetail']) is always coming as undefined . However console.log inside getDetail function print the result.
I am newbie to aws lambda with node js .. Any help will be appreciated .
Thanks in Advance

I just got solution to my question . So i am posting it,may be it will help someone else facing same issue
var notification = {
TableName: "Tablename",
KeyConditionExpression:"#userId =:userid",
Limit: 12,
ScanIndexForward: false,
ExpressionAttributeNames: {
"#userId": "userId",
},
ExpressionAttributeValues: {
":userid" :userId
}
};
docClient.query(notification, function(err, result) {
if (err) {
context.succeed({success: false, message : JSON.stringify(err, null, 2),method:"notification",type:type});
}else{
var ret =[];
var results =result.Items;
if (results.length>0) {
results.forEach(function(result, i) {
console.log(i);
var param = {
TableName: "Tablename",
ProjectionExpression:"CafeName,Cafe_MainImage",
KeyConditionExpression:"CafeID =:cafeID",
ExpressionAttributeValues: {
":cafeID" :result.CafeID
}
};
docClient.query(param, function(err, CafeDetail) {
if (err) {
context.succeed({success: false, message : JSON.stringify(err, null, 2),method:"notification",type:type});
} else {
console.log(CafeDetail.Items);
CafeDetail =CafeDetail.Items;
result['CafeDetail']=CafeDetail
console.log(result);
if (i === results.length -1) { // last iteration
context.succeed({success: true,data:results, message : "Notification list",method:"notification",type:type});
}
}
});
});
}else{
context.succeed({success: true,data:result, message : "Notification list",method:"notification",type:type});
}
}
});

Related

How to scan AWS DynamoDB with wild card in Nodejs

I use aws-sdk in nodejs, And I should scan database with wildcard.
I tried this with aws developer guide:
var params = {
TableName: RecipeTable,
FilterExpression: "#recipe = :recipe",
ExpressionAttributeNames:{
"#recipe": "recipe",
},
ExpressionAttributeValues: {
":recipe": request.params.recipe,
}
};
I can't reach the answer.
Can someone help me?
Thanks.
I found the answer: Using contains keyword.
app.get('/recipe/:recipe', (request, response) => {
var params = {
TableName: RecipeTable,
FilterExpression: "contains(#recipe, :recipe)", // Here!
ExpressionAttributeNames:{
"#recipe": "ingredients"
},
ExpressionAttributeValues: {
":recipe": request.params.recipe
}
};
result = [];
docClient.scan(params, onScan);
function onScan(err, data) {
if (!err) {
data.Items.forEach((itemdata) => {
result.push(itemdata);
});
if(typeof data.LastEvaluatedKey != "undefined") {
params.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.scan(params, onScan);
} else {
response.send(JSON.stringify({"result" : result}));
}
}
}
})
I have referenced this document.
Thanks.

Data Item not getting inserted in DynamoDB Table from AWS Lambda Trigger

I have this Lambda code setup in JS for inserting a row in DynamoDB but the code seems to have some issue and doesn't trigger.Unable to figure out the issue as cloud watch also does captures anything. The API Gateway triggers the Lambda will inserts a row in DynamoDB. Tried debugging with console and logs also but unable to figure out the actual issue that is causing the code to break.
exports.handler = async (event) => {
console.log(event);
var eventBody = JSON.parse(event.body);
var eventType = eventBody.event_type;
var content = eventBody.content;
if (content.subscription) {
var subscription = content.subscription;
switch (eventType) {
case "subscription_created":
// Add new entry to License table
console.log('subscription created event');
var customer = content.customer;
var findUserScan = {
ExpressionAttributeValues: {
':email': { S: customer.email },
},
FilterExpression: 'email = :email',
TableName: _dynamodb_userprofile_table
};
try {
await dynamodb.scan(findUserScan, function (err, data) {
if (err) {
console.log("Error", err);
} else {
var userProfileAWS;
data.Items.forEach(function (userProfile, index, array) {
userProfileAWS = AWS.DynamoDB.Converter.unmarshall(userProfile);
});
var companyName;
if (userProfileAWS) {
companyName = userProfileAWS.organization;
}
try {
var licenseEntry = {
"subscriptionID": {
S: subscription.id
},
"companyName": {
S: companyName || ""
},
"customerEmail": {
S: customer.email
},
};
await dynamodb.putItem({
"TableName": _dynamodb_license_table,
"Item": licenseEntry
}).promise()
} catch (error) {
console.log(error)
throw new Error(`Error in dynamoDB: ${JSON.stringify(error)}`);
}
}
}).promise();
} catch (error) {
throw new Error(`Error in dynamoDB: ${JSON.stringify(error)}`);
}
break;

How to Improve the Performance of mongodb long running Query

I am new to mongoDb, i am trying to update the fields for Each Records for around 10 to 15k records. When i am trying to update the records the query blocks whole database, the Running query will allow me to do any read or write operations till the Query Execution completes, is there anyway to improve the performance for this kind of Queries.
Here is My Code:
var ExisitingData=[{"isActive" : true,
"barcode" : "8908001921015",
"mrp" : 2000,
},
{"isActive" : true,
"barcode" : "7808001921019",
"mrp" : 1000,
}
....15k]
var updatedRsult=[];
async.forEach(ExisistingData, function (item, innerCallback) {
exports.populateData(item, function (err, populatedResult) {
if (err) {
innerCallback(err);
}
if(populatedResult==true)
totalRecordsUpdated++;
else
totalIgnoredRecords++;
innerCallback();
});
}, function (err) {
console.log("FinalDone");
var h1={}
h1['totalRecordsUpdated'] = totalRecordsUpdated;
h1['totalIgnoredRecords'] = totalIgnoredRecords;
updatedResult.push(h1);
updateCheck(null, updatedResult);
});
exports.populateData=function(item, mainCallback) {
var updated = false;
async.parallel([
function (callback1) {
Test.update({
$and: [
{'barcode': item['barcode']},
{'mrp': {$lt: parseInt(item['mrp'])}}
]
}, {$set: {'mrp': parseInt(item['mrp'])}}, function (err, result) {
if (err) {
console.log(err);
callback1();
}
else {
if (result['nModified'] == 1) {
console.log("Its Updated");
console.log(item);
updated=true;
callback1()
}
else {
callback1()
}
}
});
}
], function done(err) {
mainCallback(null,updated);
});
};

Confused on querying DynamoDB

I've the below data in my DynamoDB.
and I'm trying to achieve the below result.
scan through the table and get the rows where the management is NULL and Location is Midwest.
I initially tried the below query to match the Null.
var scanningParameters = {
TableName: 'LOB',
FilterExpression: "#mgmt contains NULL",
ExpressionAttributeNames: {
"#mgmt": "Management",
}
};
docClient.scan(scanningParameters, onScan);
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
} else {
// print all the movies
console.log("Scan succeeded.");
data.Items.forEach(function (data) {
console.log(
data.lineofbusiness + " name : ",
data.name);
});
if (typeof data.LastEvaluatedKey != "undefined") {
console.log("Scanning for more...");
scanningParameters.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.scan(scanningParameters, onScan);
}
}
}
I get the exception as
{
"message": "Invalid FilterExpression: Syntax error; token: \"contains\", near: \"#mgmtcontains NULL\"",
"code": "ValidationException",
"time": "2017-05-03T13:21:11.611Z",
"requestId": "0T0GU59HRJ24P96D42H9QNC97RVV4KQNSO5AEMVJF66Q9ASUAAJG",
"statusCode": 400,
"retryable": false,
"retryDelay": 13.73953651636839
}
please let me know where am I going wrong and how can I fix this.
Here is the scan item for "NULL" value (i.e. String attribute having NULL as data).
I assume that Management attribute is of String data type containing the string value "NULL".
Code:-
var AWS = require("aws-sdk");
var creds = new AWS.Credentials('akid', 'secret', 'session');
AWS.config.update({
region: "us-west-2",
endpoint: "http://localhost:8000",
credentials : creds
});
var docClient = new AWS.DynamoDB.DocumentClient();
var params = {
TableName: "lob",
FilterExpression: "#mgmt = :mgmtVal",
ExpressionAttributeNames: {
"#mgmt": "Management",
},
ExpressionAttributeValues : {
":mgmtVal" : "NULL"
}
};
docClient.scan(params, onScan);
var count = 0;
function onScan(err, data) {
if (err) {
console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
} else {
console.log("Scan succeeded.");
data.Items.forEach(function(itemData) {
console.log("Item :", ++count,JSON.stringify(itemData));
});
if (typeof data.LastEvaluatedKey != "undefined") {
params.ExclusiveStartKey = data.LastEvaluatedKey;
docClient.scan(params, onScan);
}
}
}

node js mongo db dependencies (doc not being found)

I have the following code:
var method = PushLoop.prototype;
var agent = require('./_header')
var request = require('request');
var User = require('../models/user_model.js');
var Message = require('../models/message_model.js');
var async = require('async')
function PushLoop() {};
method.startPushLoop = function() {
getUserList()
function getUserList() {
User.find({}, function(err, users) {
if (err) throw err;
if (users.length > 0) {
getUserMessages(users)
} else {
setTimeout(getUserList, 3000)
}
});
}
function getUserMessages(users) {
// console.log("getUserMessages")
async.eachSeries(users, function (user, callback) {
var params = {
email: user.email,
pwd: user.password,
token: user.device_token
}
messageRequest(params)
callback();
}, function (err) {
if (err) {
console.log(err)
setTimeout(getUserList, 3000)
}
});
}
function messageRequest(params) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ params.email +"&api_password="+ params.pwd +"&method=getSMS&type=1&limit=5"
request(url, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
if (responseObject["status"] == "success") {
async.eachSeries(messages, function(message, callback){
console.log(params.token)
saveMessage(message, params.token)
callback();
}, function(err) {
if (err) {
console.log(err)
}
// setTimeout(getUserList, 3000)
})
} else {
// setTimeout(getUserList, 3000)
}
} else {
console.log(err)
// setTimeout(getUserList, 3000)
}
});
setTimeout(getUserList, 3000)
}
function saveMessage(message, token) {
// { $and: [ { price: { $ne: 1.99 } }, { price: { $exists: true } }
// Message.find({ $and: [{ message_id: message.id}, {device_token: token}]}, function (err, doc){
Message.findOne({message_id: message.id}, function (err, doc){
if (!doc) {
console.log('emtpy today')
var m = new Message({
message_id: message.id,
did: message.did,
contact: message.contact,
message: message.message,
date: message.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: token
});
m.save(function(e) {
if (e) {
console.log(e)
} else {
agent.createMessage()
.device(token)
.alert(message.message)
.set('contact', message.contact)
.set('did', message.did)
.set('id', message.id)
.set('date', message.date)
.set('message', message.message)
.send();
}
});
}
}) //.limit(1);
}
};
module.exports = PushLoop;
Which actually works perfectly fine in my development environment - However in production (i'm using Openshift) the mongo documents get saved in an endless loop so it looks like the (if (!doc)) condition always return true therefore the document gets created each time. Not sure if this could be a mongoose issue - I also tried the "find" method instead of "findOne". My dev env has node 0.12.7 and Openshift has 0.10.x - this could be the issue, and i'm still investigating - but if anybody can spot an error I cannot see in my logic/code please let me know
thanks!
I solved this issue by using a "series" like pattern and using the shift method on the users array. The mongoose upsert findOneOrCreate is good however if there is a found document, the document is returned, if one isn't found and therefore created, it's also returned. Therefore I could not distinguish between the newly insert doc vs. a found doc, so used the same findOne function which returns null if no doc is found I just create it and send the push notification. Still abit ugly, and I know I could have used promises or the async lib, might refactor in the future. This works for now
function PushLoop() {};
var results = [];
method.go = function() {
var userArr = [];
startLoop()
function startLoop() {
User.find({},function(err, users) {
if (err) throw err;
users.forEach(function(u) {
userArr.push(u)
})
function async(arg, callback) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ arg.email +"&api_password="+ arg.password +"&method=getSMS&type=1&limit=5"
request.get(url, {timeout: 30000}, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
var status = responseObject.status
if (status === "success") {
messages.forEach(function(m) {
var message = new Message({
message_id: m.id,
did: m.did,
contact: m.contact,
message: m.message,
date: m.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: arg.device_token
});
var query = { $and : [{message_id: m.id}, {device_token: arg.device_token}] }
var query1 = { message_id: m.id }
Message.findOne(query).lean().exec(function (err, doc){
if (!doc || doc == null) {
message.save(function(e) {
console.log("message saved")
if (e) {
console.log("there is an error")
console.log(e)
} else {
console.log(message.device_token)
var messageStringCleaned = message.message.toString().replace(/\\/g,"");
var payload = {
"contact" : message.contact,
"did" : message.did,
"id" : message.message_id,
"date" : message.date,
"message" : messageStringCleaned
}
var note = new apns.Notification();
var myDevice = new apns.Device(message.device_token);
note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now.
note.badge = 3;
note.alert = messageStringCleaned;
note.payload = payload;
apnsConnection.pushNotification(note, myDevice);
}
})
}
});
});
}
else {
console.log(err)
}
}
});
setTimeout(function() {
callback(arg + "testing 12");
}, 1000);
}
// Final task (same in all the examples)
function series(item) {
if(item) {
async( item, function(result) {
results.push(result);
return series(userArr.shift());
});
} else {
return final();
}
}
function final() {
console.log('Done');
startLoop();
}
series(userArr.shift())
});
}
}
module.exports = PushLoop;

Resources